1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 anyhow::ensure!(
1443 state
1444 .base_sync_recovery
1445 .stop
1446 .as_ref()
1447 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449 );
1450 let fetched =
1451 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452 anyhow::ensure!(
1453 fetched.ok(),
1454 "cannot authorize base recovery without a fetched base"
1455 );
1456 let approved_tip = git::rev_parse(
1457 &state.repo,
1458 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459 )
1460 .await?;
1461 if let Some(control) = control.as_deref_mut() {
1462 let queue = crate::queue::Queue::at(home.join("queue"));
1463 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464 if let Some(prior) = state
1465 .base_sync_recovery
1466 .authorizations
1467 .iter()
1468 .find(|a| a.request.as_deref() == Some(&control.key()))
1469 {
1470 anyhow::ensure!(
1471 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472 "recovery request was consumed or invalidated; it cannot queue again"
1473 );
1474 return control.enqueue(&queue);
1475 }
1476 }
1477 if resumption {
1478 anyhow::ensure!(
1479 crate::daemon::compatible_run_instruction(&state, &queued),
1480 "resumption instruction does not match the retained run"
1481 );
1482 let approval = queued
1483 .resumption
1484 .as_ref()
1485 .context("no operator resumption")?;
1486 anyhow::ensure!(
1487 approval.instruction == queued.instruction,
1488 "resumption approval does not match the current task instruction"
1489 );
1490 state.authorize_base_sync_resumption(task, approval)?;
1491 } else {
1492 state.authorize_base_sync(task)?;
1493 }
1494 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495 state
1496 .base_sync_recovery
1497 .authorizations
1498 .last_mut()
1499 .unwrap()
1500 .scope = Some(crate::base_sync_recovery::Scope {
1501 tip: approved_tip,
1502 instruction: queued.instruction.clone(),
1503 episode,
1504 });
1505 if let Some(control) = control.as_deref_mut() {
1506 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507 authorization.request = Some(control.key());
1508 authorization.generation = control.generation();
1509 authorization.decision = control.decision();
1510 authorization.question = control
1511 .request
1512 .decision
1513 .as_ref()
1514 .map(|_| control.request.request.clone());
1515 }
1516 state.event(
1517 "base-recovery",
1518 "owner authorized a bounded base synchronization interval; fresh verification required",
1519 );
1520 state.save_under(home)?;
1521 if let Some(control) = control {
1522 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523 }
1524 Ok(())
1525 }
1526
1527 pub fn resume(id: &str) -> Result<Self> {
1529 let state = RunState::load(id)?;
1530 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531 if state.startup.as_ref().is_some_and(|s| !s.complete)
1532 || (state.status == RunStatus::Prep
1533 && state.instruction.trim().is_empty()
1534 && state.candidates.is_empty()
1535 && state.prep_complete == Some(false))
1536 {
1537 bail!(
1538 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539 );
1540 }
1541 crate::pr_head::ensure_owner(&state)?;
1542 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543 bail!(
1544 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545 );
1546 }
1547 if let Some(to) = &state.released_to {
1548 bail!(
1549 "run {} cannot be resumed: its worktree was released to run {}",
1550 state.short(),
1551 crate::run::short_of(to)
1552 );
1553 }
1554 let roles = state.config.resolve_roles()?;
1555 if let Some(id) = &state.config.graph.selected_implementer
1556 && state.candidates.iter().any(|c| c.agent != *id)
1557 {
1558 bail!(
1559 "replacement implementer pin does not match the saved candidates; task must stay held"
1560 );
1561 }
1562 let max_parallel = state.config.graph.max_parallel.max(1);
1563 Ok(Self {
1564 state,
1565 roles,
1566 sem: Arc::new(Semaphore::new(max_parallel)),
1567 pause: Pause::new(),
1568 interrupt: Pause::new(),
1569 })
1570 }
1571
1572 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579 Box::pin(self.execute_inner())
1582 }
1583
1584 async fn execute_inner(&mut self) -> Result<()> {
1585 anyhow::ensure!(
1586 self.state.startup.as_ref().is_none_or(|s| s.complete),
1587 "startup incomplete; inspect retained evidence before explicit recovery"
1588 );
1589 if let Some(selection) = &self.state.pr_recovery
1590 && (selection.source_proof.is_some() || selection.source_run.is_some())
1591 {
1592 Box::pin(selection.verify_source(&self.state.repo)).await?;
1593 }
1594 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597 || self.state.land_approval.is_some()
1598 || self.state.merge_decision.is_some()
1599 || !self.state.merge_repairs.is_empty()
1600 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601 {
1602 let claim = FixClaim::acquire(&self.state.dir())?;
1603 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604 let disk = RunState::load(&self.state.id)?;
1605 let checkpoint = |s: &RunState| {
1606 (
1607 s.merge_repairs.len(),
1608 s.merge_repairs
1609 .last()
1610 .map(|r| (r.question.clone(), r.phase)),
1611 s.land_approval
1612 .as_ref()
1613 .map(|a| (a.question.clone(), a.head.clone())),
1614 )
1615 };
1616 if checkpoint(&disk) != checkpoint(&self.state) {
1617 bail!(
1618 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619 );
1620 }
1621 Some(claim)
1622 } else {
1623 None
1624 };
1625 let mut execution = self.claim_driver().await?;
1626 let runner = &mut *execution.runner;
1627 let wait = runner.repository_wait();
1628 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631 let result = match result {
1632 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633 record_repository_block(&mut runner.state, &e);
1634 runner.park_here()?;
1635 Ok(())
1636 }
1637 other => other,
1638 };
1639 execution.driver.finish(&mut runner.state).await?;
1640 let ended = if result.is_err() {
1641 Some({
1642 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643 if let Err(error) = &result {
1644 notice.message = format!(
1645 "Run {} stopped with an error: {}",
1646 runner.state.id,
1647 error.to_string().lines().next().unwrap_or("unknown error")
1648 );
1649 }
1650 notice
1651 })
1652 } else {
1653 crate::notices::run_ended(&runner.state)
1654 };
1655 if let Some(notice) = ended {
1656 crate::notices::raise(notice);
1657 }
1658 result
1659 }
1660
1661 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663 }
1664
1665 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666 let wait = self.repository_wait();
1667 coordinate_operation(&mut self.state, cwd, &wait).await
1668 }
1669
1670 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672 let owner = match crate::pr_head::owner_lock_wait(
1673 &crate::run::home(),
1674 &self.state.id,
1675 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676 )
1677 .await
1678 {
1679 Ok(owner) => owner,
1680 Err(error) => {
1681 let detail = format!("worktree maintenance or ownership update: {error:#}");
1682 self.state.resource_block = Some(crate::run::ResourceBlock {
1686 cause: crate::run::ResourceBlockCause::Other,
1687 node: "start".into(),
1688 kind: "worktree maintenance".into(),
1689 detail: detail.clone(),
1690 at: Timestamp::now(),
1691 });
1692 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693 }
1694 };
1695 let disk = RunState::load(&self.state.id)?;
1696 crate::pr_head::ensure_owner(&disk)?;
1697 let claimed =
1698 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699 anyhow::ensure!(
1704 disk.liveness(claimed) != crate::run::Liveness::Live,
1705 "run already has a live driver; wait for its execution to end"
1706 );
1707 if let Some(to) = &disk.released_to {
1708 bail!(
1709 "run {} cannot continue: its worktree was released to run {}",
1710 self.state.short(),
1711 crate::run::short_of(to)
1712 );
1713 }
1714 anyhow::ensure!(
1715 serde_json::to_value(&disk.base_sync_recovery)?
1716 == serde_json::to_value(&self.state.base_sync_recovery)?,
1717 "base recovery changed since this run was loaded; reload the saved run"
1718 );
1719 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720 let mut execution = DriverExecution {
1721 runner: self,
1722 driver,
1723 };
1724 let _owner = owner;
1727 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728 .await?;
1729 let state = &mut execution.runner.state;
1730 state.parked = false;
1735 state.resource_wait = None;
1736 state.resource_block = None;
1737 state.clear_active();
1744 let pid = std::process::id();
1759 state.driver_pid = Some(pid);
1760 state.driver_started_at = crate::proc::process_started_at(pid);
1761 state.driver_exited = false;
1762 state.driver_claimed = Some(true);
1763 state.begin_pass();
1764 execution.driver.bind(state);
1765 state.save()?;
1766 execution.driver.record(state)?;
1767 Ok(execution)
1768 }
1769
1770 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774 if let Err(error) = &result
1775 && !error.is::<crate::repo_lock::ResourceBlocked>()
1776 && self
1777 .state
1778 .repair_grants
1779 .last()
1780 .is_some_and(|g| g.conflict.is_none())
1781 {
1782 self.state.repair_grants.last_mut().unwrap().phase =
1783 crate::repair_budget::Phase::Complete;
1784 self.state.status = RunStatus::Blocked;
1785 self.state.save()?;
1786 bail!(
1787 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788 self.state.passes.last().map_or(0, |p| p.index)
1789 );
1790 }
1791 result
1792 }
1793
1794 async fn execute_granted_repair(&mut self) -> Result<()> {
1797 use crate::repair_budget::{Kind, Phase};
1798 let index = self.state.repair_grants.len() - 1;
1799 let winner = self
1800 .state
1801 .winner()
1802 .cloned()
1803 .context("repair winner missing")?;
1804 loop {
1805 let grant = self.state.repair_grants[index].clone();
1806 match grant.phase {
1807 Phase::Pending => {
1808 if grant.spent >= grant.receipt.request.rounds {
1809 self.state.repair_grants[index].phase = Phase::Complete;
1810 self.state.status = RunStatus::Blocked;
1811 self.state.save()?;
1812 return Ok(());
1813 }
1814 self.state.repair_grants[index].spent += 1;
1817 self.state.repair_grants[index].phase = Phase::Fixing;
1818 self.state.save()?;
1819 match grant.receipt.request.kind {
1820 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821 Kind::Gate => {
1822 let failed = self.state.gate.clone();
1823 let _ = self.gate_fix_round(&winner, &failed).await?;
1824 }
1825 Kind::Review => {
1826 let mut record = self
1827 .state
1828 .reviews
1829 .last()
1830 .cloned()
1831 .context("review failure missing")?;
1832 let round = record.round;
1833 let step = self.run_fix_round(&winner, &mut record).await?;
1834 self.state.reviews[round - 1] = record;
1835 if matches!(step, FixStep::Recorded { report_lost: true }) {
1836 self.state.repair_grants[index].phase = Phase::Complete;
1837 self.state.status = RunStatus::Blocked;
1838 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839 self.state.save()?;
1840 return Ok(());
1841 }
1842 if matches!(step, FixStep::Resource) {
1843 self.stop_for_resource()?;
1844 return Ok(());
1845 }
1846 self.state.save()?;
1847 }
1848 }
1849 if self.resource_halt()? {
1850 return Ok(());
1851 }
1852 }
1853 Phase::Fixing => {
1854 if grant.receipt.request.kind == Kind::BaseConflict {
1855 self.repair_base_conflict(false).await?;
1856 }
1857 let _coordination = self.coordinate(&winner.worktree).await?;
1860 let rescued = git::rescue_commit(
1861 &winner.worktree,
1862 "magi: preserve interrupted granted repair",
1863 )
1864 .await?;
1865 self.state.note_withheld("repair-grant", &rescued.withheld);
1866 self.state
1869 .verification_history
1870 .push(crate::run::VerificationEvidence {
1871 superseded_by: None,
1872 archived_at: Some(Timestamp::now()),
1873 fixes_spent: Some(self.state.gate_fixes.len()),
1874 gate_head: self.state.gate_head.take(),
1875 gate_ran: std::mem::take(&mut self.state.gate_ran),
1876 gate: std::mem::take(&mut self.state.gate),
1877 e2e: self.state.head_e2e.take(),
1878 });
1879 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880 self.state
1881 .verification_history
1882 .push(crate::run::VerificationEvidence {
1883 superseded_by: None,
1884 archived_at: Some(Timestamp::now()),
1885 fixes_spent: Some(self.state.gate_fixes.len()),
1886 gate_head: None,
1887 gate_ran: false,
1888 gate: Vec::new(),
1889 e2e: Some(e2e),
1890 });
1891 }
1892 self.state.repair_grants[index].review_start = self.state.reviews.len();
1893 self.state.repair_grants[index].head =
1894 git::rev_parse(&winner.worktree, "HEAD").await?;
1895 self.state.repair_grants[index].phase = Phase::Reviewing;
1896 if grant.receipt.request.kind == Kind::BaseConflict {
1897 self.state.base_sync_recovery.stop = None;
1898 if let Some(sync) = &mut self.state.base_sync {
1899 sync.tip = grant
1900 .receipt
1901 .request
1902 .tip
1903 .clone()
1904 .context("repair base tip missing")?;
1905 sync.conflict = None;
1906 sync.behind = 0;
1907 }
1908 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909 }
1910 self.state.save()?;
1911 }
1912 Phase::Reviewing => {
1913 self.review_loop_pass().await?;
1914 if self.resource_halt()? {
1915 return Ok(());
1916 }
1917 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918 let clean = self.state.reviews.len() > grant.review_start
1919 && self.state.reviews.last().is_some_and(|r| {
1920 r.head == head
1921 && r.clean
1922 && r.expected > 0
1923 && r.answered == r.expected
1924 && matches!(
1925 r.e2e_status(),
1926 E2eStatus::Passed | E2eStatus::NotConfigured
1927 )
1928 });
1929 if !clean {
1930 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932 == Kind::Review
1933 && grant.spent < grant.receipt.request.rounds
1934 && self.state.reviews.last().is_some_and(|r| {
1935 r.fix.is_none()
1936 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937 }) {
1938 Phase::Pending
1939 } else {
1940 Phase::Complete
1941 };
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "repair-grant",
1945 "fresh review did not clear the repaired head; no publication",
1946 );
1947 self.state.save()?;
1948 if self.state.repair_grants[index].phase == Phase::Pending {
1949 continue;
1950 }
1951 return Ok(());
1952 }
1953 self.state.repair_grants[index].phase = Phase::Gating;
1954 self.state.save()?;
1955 }
1956 Phase::Gating => {
1957 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958 self.state.reconcile_verification_head(&head);
1959 let wait = self.repository_wait();
1960 if self
1961 .state
1962 .reviews
1963 .last()
1964 .is_none_or(|r| r.head != head || !r.clean)
1965 {
1966 self.state.status = RunStatus::Blocked;
1967 self.state.repair_grants[index].phase = Phase::Complete;
1968 self.state.event(
1969 "repair-grant",
1970 "head changed since fresh review; repair held",
1971 );
1972 self.state.save()?;
1973 return Ok(());
1974 }
1975 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976 return Ok(());
1977 }
1978 let outcomes = self.run_gate(&winner).await?;
1979 if verify_inconclusive(&outcomes) {
1980 return Ok(());
1981 }
1982 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983 if after != head {
1984 self.state.repair_grants[index].review_start = self.state.reviews.len();
1985 self.state.repair_grants[index].head = after;
1986 self.state.repair_grants[index].phase = Phase::Reviewing;
1987 self.state.status = RunStatus::Blocked;
1988 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let passed = outcomes.iter().all(CommandOutcome::ok);
1993 if passed
1996 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997 {
1998 return Ok(());
1999 }
2000 self.state.repair_grants[index].succeeded = passed;
2001 self.state.gate = outcomes;
2002 self.state.gate_ran = true;
2003 self.state.status = if passed {
2004 RunStatus::Gating
2005 } else {
2006 RunStatus::Blocked
2007 };
2008 self.state.repair_grants[index].phase = if !passed
2009 && grant.receipt.request.kind == Kind::Gate
2010 && grant.spent < grant.receipt.request.rounds
2011 {
2012 Phase::Pending
2013 } else {
2014 Phase::Complete
2015 };
2016 if passed && self.state.config.merge.mode == MergeMode::Pr {
2017 self.state.pr_publication_pending = true;
2019 } else if passed {
2020 self.state.status = RunStatus::Ready;
2021 }
2022 self.state.save()?;
2023 if !passed {
2024 continue;
2025 }
2026 if self.state.config.merge.mode != MergeMode::Pr {
2030 self.state.status = RunStatus::Ready;
2031 self.state.event(
2032 "repair-grant",
2033 "repair verified locally; grant carries no merge authorization",
2034 );
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 return self.merge().await;
2039 }
2040 Phase::Complete => return Ok(()),
2041 }
2042 }
2043 }
2044
2045 async fn execute_graph(&mut self) -> Result<()> {
2046 let archive_home = crate::run::home().canonicalize()?;
2050 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051 let repo = self.state.repo.clone();
2052 let _coordination = self.coordinate(&repo).await?;
2053 ensure_setup_config(&mut self.state, &repo).await?;
2054 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055 }
2056 self.state.configure_recovery_resume()?;
2057 if self
2058 .state
2059 .repair_grants
2060 .last()
2061 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062 {
2063 return self.execute_granted_repair().await;
2064 }
2065 if self
2066 .state
2067 .repair_grants
2068 .last()
2069 .is_some_and(|g| !g.succeeded)
2070 {
2071 self.state.status = RunStatus::Blocked;
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.persist_candidates().await?;
2078 if self.state.base_sync_blocked() {
2081 self.sync_to_base().await?;
2082 if !self.state.base_sync_blocked()
2083 && self.state.status != RunStatus::AlreadyInBase
2084 && !self.resource_halt()?
2085 {
2086 self.finish_after_tally().await?;
2087 }
2088 return Ok(());
2089 }
2090 if self
2091 .state
2092 .merge_decision
2093 .as_ref()
2094 .is_some_and(|decision| !decision.closed)
2095 && self.state.status != RunStatus::Landing
2096 {
2097 if crate::merge_decision::resumable_hold(&self.state) {
2098 self.state.merge = Some(MergeOutcome {
2099 mode: MergeMode::Pr,
2100 ok: false,
2101 detail: self
2102 .state
2103 .merge_decision
2104 .as_ref()
2105 .unwrap()
2106 .target
2107 .pr
2108 .clone(),
2109 empty: false,
2110 });
2111 self.run_land().await?;
2112 self.settle_questions();
2113 return Ok(());
2114 }
2115 bail!(
2116 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117 );
2118 }
2119 if self.state.config.graph.selected_implementer.is_some()
2120 && self.state.status == RunStatus::Blocked
2121 && self.state.tally.is_none()
2122 && self.state.candidate_preservation.is_empty()
2123 && self.state.interrupted_implementations.is_empty()
2124 {
2125 bail!(
2126 "selected implementation is held; explicitly release the owning task to retry it"
2127 );
2128 }
2129
2130 if self.state.merge_repairs.last().is_some_and(|r| {
2131 matches!(
2132 r.phase,
2133 crate::run::MergeRepairPhase::Requested
2134 | crate::run::MergeRepairPhase::Fixing
2135 | crate::run::MergeRepairPhase::Reviewing
2136 )
2137 }) {
2138 if self.repair_merge_review().await? {
2139 self.run_land().await?;
2140 }
2141 self.settle_questions();
2142 return Ok(());
2143 }
2144 if self.state.pr_publication_pending {
2147 self.merge().await?;
2148 if self.resource_halt()? {
2149 return Ok(());
2150 }
2151 self.state.save()?;
2152 self.settle_questions();
2153 return Ok(());
2154 }
2155 if self.state.status == RunStatus::Stalled {
2168 if self.recover_stall().await? {
2169 self.finish_after_tally().await?;
2170 } else {
2171 self.state.save()?;
2173 }
2174 return Ok(());
2175 }
2176 if self.state.status == RunStatus::Landing
2186 || crate::handover::landing_checkpoint(&self.state)
2187 {
2188 self.state.land_sync_pending = true;
2189 self.state.save()?;
2190 self.run_land().await?;
2191 self.settle_questions();
2196 return Ok(());
2197 }
2198 if self.state.github_text.is_some()
2203 && self.state.merge.is_none()
2204 && self.state.status == RunStatus::Gating
2205 {
2206 self.merge().await?;
2207 if self.resource_halt()? {
2208 return Ok(());
2209 }
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213 if let Err(e) = self.prep().await {
2214 if e.is::<crate::repo_lock::ResourceBlocked>() {
2215 return Err(e);
2216 }
2217 self.state.status = RunStatus::Blocked;
2218 self.state
2219 .prep_failure
2220 .get_or_insert_with(|| format!("{e:#}"));
2221 self.state
2222 .event("prep", format!("preparation stopped: {e:#}"));
2223 self.state.save()?;
2224 return Err(e);
2225 }
2226 if self.park_here()? {
2227 return Ok(());
2228 }
2229 self.advise().await?;
2230 if self.park_here()? {
2231 return Ok(());
2232 }
2233 self.implement().await?;
2234 if self.park_here()? {
2235 return Ok(());
2236 }
2237 if self.state.status == RunStatus::VerifiedNoop {
2241 return Ok(());
2242 }
2243 self.judge().await?;
2244 if self.park_here()? {
2245 return Ok(());
2246 }
2247 self.deliberate().await?;
2248 if self.park_here()? {
2249 return Ok(());
2250 }
2251 self.vote().await?;
2252 if self.park_here()? {
2253 return Ok(());
2254 }
2255 self.tally()?;
2256 if self.state.status == RunStatus::Stalled {
2261 self.state.save()?;
2265 return Ok(());
2266 }
2267 self.finish_after_tally().await?;
2268 Ok(())
2269 }
2270
2271 fn stop_for_resource(&mut self) -> Result<()> {
2276 if self.pause.parked() || self.interrupt.parked() {
2279 self.park_here()?;
2280 return Ok(());
2281 }
2282 if self.state.resource_block.is_some() {
2283 self.state.status = RunStatus::Blocked;
2284 }
2285 self.state.save()
2286 }
2287
2288 fn resource_halt(&mut self) -> Result<bool> {
2290 if self.state.resource_block.is_none() {
2291 return Ok(false);
2292 }
2293 self.stop_for_resource()?;
2294 Ok(true)
2295 }
2296
2297 fn park_here(&mut self) -> Result<bool> {
2304 let parking = self.pause.parked() || self.interrupt.parked();
2305 if !parking && self.state.resource_block.is_some() {
2306 self.stop_for_resource()?;
2307 return Ok(true);
2308 }
2309 if !self.pause.parked() && !self.interrupt.parked() {
2314 return Ok(false);
2315 }
2316 if self.state.parked {
2317 return Ok(true);
2318 }
2319 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320 if let Some(block) = &mut self.state.resource_block
2321 && matches!(
2322 block.cause,
2323 crate::run::ResourceBlockCause::Cancelled { .. }
2324 )
2325 {
2326 block.cause = crate::run::ResourceBlockCause::Cancelled {
2327 park: cause.clone(),
2328 };
2329 }
2330 let why = match cause.as_ref().map(ParkCause::label) {
2331 Some(reason) => format!(
2332 "parked after `{}` ({reason}) — resume to carry on from here",
2333 self.state.status.as_str()
2334 ),
2335 None => format!(
2336 "parked after `{}` — resume to carry on from here",
2337 self.state.status.as_str()
2338 ),
2339 };
2340 self.state.event("park", why);
2341 self.state
2342 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343 self.state.parked = true;
2344 self.state.save()?;
2345 Ok(true)
2346 }
2347
2348 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350 if self.state.prep_failure.is_none() {
2351 return Ok(());
2352 }
2353 let failed_at = self
2354 .state
2355 .events
2356 .iter()
2357 .rev()
2358 .find(|event| event.node == "prep")
2359 .map_or(self.state.updated_at, |event| event.at);
2360 anyhow::ensure!(
2361 approval_at > failed_at,
2362 "preparation retry requires a new explicit release after the failure"
2363 );
2364 self.state.prep_failure = None;
2365 self.state.prep_complete = Some(false);
2366 self.state
2367 .event("prep-retry", "operator approved retry of preparation");
2368 Ok(())
2369 }
2370
2371 pub fn on_pause(&mut self, pause: Pause) {
2373 self.pause = pause;
2374 }
2375
2376 pub fn watch_interrupt(&mut self, pause: Pause) {
2382 self.interrupt = pause;
2383 }
2384
2385 fn settle_questions(&mut self) {
2405 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407 }
2408 }
2409
2410 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413 Box::pin(self.finish_after_tally_inner())
2414 }
2415
2416 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417 self.graft().await?;
2420 if self.resource_halt()? {
2421 return Ok(());
2422 }
2423 self.fold_losers().await?;
2424 self.sync_to_base().await?;
2429 if self.hold_base_sync().await? || self.resource_halt()? {
2430 return Ok(());
2431 }
2432 if self.state.status == RunStatus::AlreadyInBase {
2433 return Ok(());
2434 }
2435 self.review_loop().await?;
2436 if self.resource_halt()? {
2437 return Ok(());
2438 }
2439 self.sync_to_base().await?;
2440 if self.hold_base_sync().await? || self.resource_halt()? {
2441 return Ok(());
2442 }
2443 if self.state.status == RunStatus::AlreadyInBase {
2444 return Ok(());
2445 }
2446 self.gate().await?;
2447 self.merge().await?;
2448 if self.resource_halt()? {
2449 return Ok(());
2450 }
2451 self.state.save()?;
2452 Ok(())
2453 }
2454
2455 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462 let result = match ensure_setup_config(&mut self.state, repo).await {
2463 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464 Err(e) => Err(e),
2465 };
2466 if let Err(e) = result {
2467 git::worktree_remove_clean(repo, wt).await.ok();
2468 if in_prep {
2469 self.rollback_prep(repo).await;
2470 }
2471 self.state.status = RunStatus::Blocked;
2472 self.state.event(
2473 "setup",
2474 format!("worktree setup failed in {}: {e:#}", wt.display()),
2475 );
2476 self.state.save()?;
2477 return Err(e);
2478 }
2479 Ok(())
2480 }
2481
2482 async fn rollback_prep(&mut self, repo: &Path) {
2484 let root = self.state.worktree_root();
2485 for c in std::mem::take(&mut self.state.candidates) {
2486 if matches!(
2487 git::worktree_remove_clean(repo, &c.worktree).await,
2488 Ok(true)
2489 ) {
2490 git::branch_delete(repo, &c.branch).await.ok();
2491 } else {
2492 self.state.candidates.push(c);
2493 }
2494 }
2495 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496 for j in 1..=seats {
2497 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2498 .await
2499 .ok();
2500 }
2501 for k in 1..=self.state.config.graph.advisors {
2502 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2503 .await
2504 .ok();
2505 }
2506 }
2507
2508 async fn prep(&mut self) -> Result<()> {
2509 if let Some(reason) = &self.state.prep_failure {
2510 bail!(
2511 "candidate preparation previously failed: {reason}; correct the source before retrying"
2512 );
2513 }
2514 if self
2515 .state
2516 .prep_complete
2517 .unwrap_or(!self.state.candidates.is_empty())
2518 {
2519 return Ok(());
2520 }
2521 let repo = self.state.repo.clone();
2522 let _coordination = self.coordinate(&repo).await?;
2525 let base = self.state.base_commit.clone();
2526 let plan = self.state.sources.plan(&repo, &base).await?;
2527 self.rollback_prep(&repo).await;
2528 if !self.state.candidates.is_empty() {
2529 bail!(
2530 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531 );
2532 }
2533 self.state.prep_complete = Some(false);
2534 self.state.status = RunStatus::Prep;
2535 self.state.save()?;
2536 let repo = self.state.repo.clone();
2537 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538 let root = self.state.worktree_root();
2539 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541 let hooks_dir = self.state.dir().join("hooks");
2544 if self.state.config.blind.commit_msg_hook {
2545 std::fs::create_dir_all(&hooks_dir)
2546 .with_context(|| format!("create {}", hooks_dir.display()))?;
2547 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548 let path = hooks_dir.join("commit-msg");
2549 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550 make_executable(&path)?;
2551 git::acquire_worktree_config(&repo).await?;
2559 self.state.enabled_worktree_config = true;
2560 }
2561
2562 ensure_setup_config(&mut self.state, &repo).await?;
2565
2566 for (index, (spec, label)) in self
2567 .roles
2568 .implementers
2569 .clone()
2570 .into_iter()
2571 .zip(labels)
2572 .enumerate()
2573 {
2574 let branch = self.state.branch_for(label);
2575 let worktree = root.join(format!("cand-{label}"));
2576 self.state.event(
2580 "prep-intent",
2581 serde_json::json!({
2582 "branch": branch,
2583 "worktree": worktree,
2584 "start": start,
2585 "picks": plan.picks,
2586 })
2587 .to_string(),
2588 );
2589 self.state.save()?;
2590 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591 if self.state.config.blind.commit_msg_hook {
2592 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593 }
2594 git::local_exclude(&worktree, "/.magi/").await?;
2595 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596 git::worktree_remove_clean(&repo, &worktree).await.ok();
2599 git::branch_delete(&repo, &branch).await.ok();
2600 self.rollback_prep(&repo).await;
2601 self.state.status = RunStatus::Blocked;
2602 self.state
2603 .event("prep", format!("worktree setup failed: {e:#}"));
2604 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605 self.state.save()?;
2606 return Err(e);
2607 }
2608 self.state.candidates.push(Candidate {
2609 index,
2610 label,
2611 agent: spec.id.clone(),
2612 branch,
2613 worktree,
2614 summary: String::new(),
2615 stat: String::new(),
2616 files: 0,
2617 commits: 0,
2618 empty: false,
2619 failed: None,
2620 verified_noop: None,
2621 duration_ms: 0,
2622 folded: false,
2623 });
2624 self.state.save()?;
2625 for pick in &plan.picks {
2626 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627 {
2628 self.state.status = RunStatus::Blocked;
2629 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630 self.state
2631 .event("prep", format!("cannot apply referenced commit: {e}"));
2632 self.state.save()?;
2633 return Err(e);
2634 }
2635 }
2636 }
2637
2638 for j in 1..=self.roles.judges.len() {
2639 let wt = root.join(format!("judge-{j}"));
2640 if !wt.exists() {
2641 git::worktree_add_detached(&repo, &wt, &base).await?;
2642 self.setup_seat_worktree(&repo, &wt, true).await?;
2643 }
2644 }
2645
2646 if self.state.config.graph.advise {
2654 for k in 1..=self.state.config.graph.advisors {
2655 let wt = root.join(format!("advisor-{k}"));
2656 if !wt.exists() {
2657 git::worktree_add_detached(&repo, &wt, &base).await?;
2658 self.setup_seat_worktree(&repo, &wt, true).await?;
2659 }
2660 }
2661 }
2662
2663 let authors: Vec<&str> = self
2668 .roles
2669 .implementers
2670 .iter()
2671 .map(|a| a.id.as_str())
2672 .collect();
2673 let overlap: Vec<String> = self
2674 .roles
2675 .judges
2676 .iter()
2677 .enumerate()
2678 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680 .collect();
2681 if !overlap.is_empty() {
2682 let note = format!(
2683 "{} also authored a candidate; blind, but the panel is less \
2684 independent than {} distinct agents would be",
2685 overlap.join(", "),
2686 self.roles.judges.len()
2687 );
2688 self.state.event("prep", note);
2689 }
2690
2691 self.state.event(
2692 "prep",
2693 format!(
2694 "{} candidates, {} judges, base {} ({})",
2695 self.state.candidates.len(),
2696 self.roles.judges.len(),
2697 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698 self.state.base_branch
2699 ),
2700 );
2701 self.state.prep_complete = Some(true);
2702 self.state.status = RunStatus::Implementing;
2703 self.state.save()?;
2704 Ok(())
2705 }
2706
2707 async fn advise(&mut self) -> Result<()> {
2740 let implement_untouched = self
2741 .state
2742 .candidates
2743 .iter()
2744 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745 if !self.state.config.graph.advise || self.state.advise_attempted {
2746 return Ok(());
2747 }
2748 if !implement_untouched {
2749 self.state.event(
2750 "advise",
2751 "skipping the design-deliberation stage: at least one \
2752 candidate already shows implementation progress, so this \
2753 run is past the point the stage exists to run before"
2754 .to_owned(),
2755 );
2756 self.state.advise_attempted = true;
2757 self.state.save()?;
2758 return Ok(());
2759 }
2760 let run_id = self.state.id.clone();
2761 let prompts = self.state.config.prompts.clone();
2762 let instruction = self.state.instruction.clone();
2763 let language = self.state.config.graph.language.clone();
2764 let root = self.state.worktree_root();
2765 let n = self.state.config.graph.advisors;
2766 let where_recorded = self.state.dir().join("run.json");
2767
2768 let seats = match self.state.config.advisors() {
2769 Ok(seats) if !seats.is_empty() => seats,
2770 Ok(_) => {
2771 self.state.event(
2772 "advise",
2773 format!(
2774 "[graph] advisors is 0; skipping the design-deliberation \
2775 stage and continuing without a synthesis brief (see {})",
2776 where_recorded.display()
2777 ),
2778 );
2779 self.state.advise_attempted = true;
2780 self.state.save()?;
2781 return Ok(());
2782 }
2783 Err(e) => {
2784 self.state.event(
2785 "advise",
2786 format!(
2787 "could not resolve advisor seats ({e:#}); continuing \
2788 without a design-deliberation brief (see {})",
2789 where_recorded.display()
2790 ),
2791 );
2792 self.state.advise_attempted = true;
2793 self.state.save()?;
2794 return Ok(());
2795 }
2796 };
2797
2798 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799 let artifacts = agent::artifacts_dir(&self.state.dir());
2800 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802 let mut jobs = Vec::new();
2803 for (i, spec) in seats.iter().cloned().enumerate() {
2804 let seat_key = format!("advisor-{}", i + 1);
2805 let seat = self.seat(&seat_key, &spec.id);
2806 jobs.push(SeatJob {
2807 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808 spec,
2809 seat,
2810 cwd: worktrees[i % worktrees.len()].clone(),
2811 timeout,
2812 allow_write: false,
2813 sessions: false,
2814 artifacts: artifacts.clone(),
2815 stem: seat_key,
2816 handover: None,
2817 });
2818 }
2819
2820 self.state.event(
2821 "advise",
2822 format!(
2823 "{} advisor seat(s) sketching a design in parallel",
2824 jobs.len()
2825 ),
2826 );
2827 let mut quota_losses = Vec::new();
2828 let cache = self.state.config.cache_dir();
2829 let ctx = WaveCtx {
2830 carry_seats: false,
2831 pauses: Some((&self.pause, &self.interrupt)),
2832 run: &run_id,
2833 node: "advise",
2834 prompts: &prompts,
2835 cache: cache.as_deref(),
2836 round: None,
2837 };
2838 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839 let results = ask_json_wave::<Proposal>(
2840 jobs,
2841 Arc::clone(&self.sem),
2842 self.state.config.graph.retries,
2843 &advisor_roster,
2844 &ctx,
2845 &mut quota_losses,
2846 &mut self.state,
2847 &|p: &Proposal| p.validate(),
2848 )
2849 .await;
2850 if self.state.resource_block.is_some() {
2851 return self.stop_for_resource();
2852 }
2853 self.state.quota.extend(quota_losses);
2854
2855 let mut records = Vec::with_capacity(results.len());
2856 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857 let agent_id = seat.agent.clone();
2858 self.state.seats.insert(seat.key.clone(), seat);
2859 match res {
2860 Ok((proposal, out)) => {
2861 self.state
2862 .event("advise", format!("advisor-{} proposed a design", i + 1));
2863 records.push(advise::AdvisorRecord::proposed(
2864 i + 1,
2865 agent_id,
2866 proposal,
2867 out.duration_ms,
2868 ));
2869 }
2870 Err(e) => {
2871 self.state.event(
2872 "advise",
2873 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874 );
2875 records.push(advise::AdvisorRecord::failed(
2876 i + 1,
2877 agent_id,
2878 e.to_string(),
2879 ));
2880 }
2881 }
2882 }
2883
2884 let mut advice = advise::Advice {
2885 records,
2886 synthesis: None,
2887 };
2888 if advice.proposals().is_empty() {
2889 self.state.event(
2890 "advise",
2891 "no advisor produced a usable proposal; continuing without a \
2892 synthesis brief"
2893 .to_owned(),
2894 );
2895 } else {
2896 match self
2897 .synthesize_brief(
2898 &advice,
2899 &instruction,
2900 &language,
2901 &worktrees[0],
2902 &artifacts,
2903 &run_id,
2904 &prompts,
2905 cache.as_deref(),
2906 )
2907 .await
2908 {
2909 Ok(Some(text)) => {
2910 self.state.event(
2911 "advise",
2912 "synthesized a design brief for the implementer".to_owned(),
2913 );
2914 advice.synthesis = Some(text);
2915 }
2916 Ok(None) => {
2917 self.state.event(
2918 "advise",
2919 "the synthesis seat produced nothing usable; continuing \
2920 without a design brief"
2921 .to_owned(),
2922 );
2923 }
2924 Err(e) => {
2925 if e.is::<crate::repo_lock::ResourceBlocked>() {
2926 return Err(e);
2927 }
2928 self.state.event(
2929 "advise",
2930 format!("could not synthesize a design brief: {e:#}"),
2931 );
2932 }
2933 }
2934 }
2935 advise::apply_reflection(&mut advice);
2936
2937 self.state.advice = Some(advice);
2938 self.state.advise_attempted = true;
2939 self.state.save()?;
2940 Ok(())
2941 }
2942
2943 #[allow(clippy::too_many_arguments)]
2955 async fn synthesize_brief(
2956 &mut self,
2957 advice: &advise::Advice,
2958 instruction: &str,
2959 language: &str,
2960 cwd: &Path,
2961 artifacts: &Path,
2962 run_id: &str,
2963 prompts: &Prompts,
2964 cache: Option<&Path>,
2965 ) -> Result<Option<String>> {
2966 let chain = agent::pick_chain(
2967 &self.state.config.agents,
2968 self.state.config.roles.synthesizer.as_ref(),
2969 &agent::installed,
2970 "synthesizer",
2971 )?;
2972 let proposals = advice.proposals();
2973 let mut prompt = prompt::with_overlay(
2974 prompt::synthesize_brief(instruction, &proposals, language),
2975 prompts.overlay("advise"),
2976 );
2977 if cache.is_some() {
2978 prompt.push('\n');
2983 prompt.push_str(&prompt::build_cache_note("advise", false));
2984 }
2985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986 let mut last = None;
2991 for (n, spec) in chain.iter().enumerate() {
2992 if n > 0 {
2993 self.state
2994 .event("advise", format!("synthesis falling back to {}", spec.id));
2995 }
2996 let mut seat = self.seat("advise-synthesis", &spec.id);
2997 let wait = self.repository_wait();
2998 let outcome = wait
2999 .scope(agent::invoke(
3000 spec,
3001 &mut seat,
3002 &Invocation {
3003 coordination: self.state.config.coordination.as_ref(),
3004 fetched_base: None,
3005 cwd,
3006 prompt: &prompt,
3007 timeout,
3008 allow_write: false,
3009 sessions: false,
3010 artifacts,
3011 stem: &if n == 0 {
3012 "advise-synthesis".to_owned()
3013 } else {
3014 format!("advise-synthesis-{}", spec.id)
3015 },
3016 run: run_id,
3017 node: "advise",
3018 cache_dir: None,
3019 attachments: &[],
3020 writable: &[],
3021 },
3022 ))
3023 .await;
3024 if outcome.is_ok() {
3025 self.state.seats.insert(seat.key.clone(), seat);
3026 }
3027 let advance = agent::chain_advances(&outcome);
3028 last = Some(outcome);
3029 if !advance {
3030 break;
3031 }
3032 }
3033 let out = last.expect("a chain holds at least one agent")?;
3036 if !out.usable() {
3037 return Ok(None);
3038 }
3039 let text =
3040 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041 Ok((!text.trim().is_empty()).then_some(text))
3042 }
3043
3044 async fn implement(&mut self) -> Result<()> {
3047 self.persist_candidates().await?;
3048 let run_id = self.state.id.clone();
3053 let prompts = self.state.config.prompts.clone();
3054 let todo: Vec<usize> = self
3055 .state
3056 .candidates
3057 .iter()
3058 .enumerate()
3059 .filter(|(_, c)| {
3060 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061 && c.failed.is_none()
3062 && !c.empty
3063 })
3064 .map(|(i, _)| i)
3065 .collect();
3066 if todo.is_empty() {
3067 return self.after_implement();
3068 }
3069 self.state.status = RunStatus::Implementing;
3070
3071 let language = self.state.config.graph.language.clone();
3072 let github_language = self.state.config.graph.github_language.clone();
3073 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074 let sessions = self.state.config.graph.sessions;
3075 let artifacts = agent::artifacts_dir(&self.state.dir());
3076 let brief = self
3080 .state
3081 .advice
3082 .as_ref()
3083 .and_then(|a| a.synthesis.as_deref())
3084 .map(str::to_owned);
3085 let attachments = self.state.attachments.clone();
3086
3087 let mut jobs = Vec::new();
3088 for &i in &todo {
3089 let (index, label, worktree) = {
3090 let c = &self.state.candidates[i];
3091 (c.index, c.label, c.worktree.clone())
3092 };
3093 let spec = self.roles.implementers[index].clone();
3094 let seat_key = format!("impl-{label}");
3095 let seat = self.seat(&seat_key, &spec.id);
3096 let instruction = seeded_instruction(&self.state);
3097 jobs.push(SeatJob {
3098 spec,
3099 seat,
3100 prompt: prompt::implement_in(
3101 &instruction,
3102 &worktree.to_string_lossy(),
3103 &language,
3104 &github_language,
3105 brief.as_deref(),
3106 &attachments,
3107 ),
3108 cwd: worktree,
3109 timeout,
3110 allow_write: true,
3111 sessions,
3112 artifacts: artifacts.clone(),
3113 stem: format!("impl-{label}"),
3114 handover: None,
3115 });
3116 }
3117
3118 self.state.event(
3119 "implement",
3120 format!("{} candidates in parallel", jobs.len()),
3121 );
3122 let mut sent = jobs.clone();
3128 let cache = self.state.config.cache_dir();
3129 let ctx = WaveCtx {
3130 carry_seats: false,
3131 pauses: Some((&self.pause, &self.interrupt)),
3132 run: &run_id,
3133 node: "implement",
3134 prompts: &prompts,
3135 cache: cache.as_deref(),
3136 round: None,
3137 };
3138 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139 if !wave_blocked(&results) {
3140 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141 .await;
3142 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143 .await?;
3144 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145 .await;
3146 }
3147 if !results.is_empty()
3150 && results
3151 .iter()
3152 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153 {
3154 let losses: Vec<_> = results
3155 .iter()
3156 .map(|(_, seat, out)| {
3157 let AgentOutcome::Quota(o) = out else {
3158 unreachable!()
3159 };
3160 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161 })
3162 .collect();
3163 self.state.quota.extend(losses.clone());
3164 for (_, seat, _) in results {
3165 self.state.seats.insert(seat.key.clone(), seat);
3166 }
3167 for &i in &todo {
3168 let label = self.state.candidates[i].label;
3169 self.state.interrupted_implementations.insert(label);
3170 self.state
3171 .candidate_preservation
3172 .insert(label, "machine outage awaiting persistence".into());
3173 }
3174 block_machine_outage(&mut self.state, "implement", &losses);
3177 self.state.save()?;
3178 self.persist_candidates().await?;
3179 for &i in &todo {
3180 self.state
3181 .interrupted_implementations
3182 .insert(self.state.candidates[i].label);
3183 }
3184 return self.stop_for_resource();
3185 }
3186 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189 && results
3190 .iter()
3191 .any(|(_, _, out)| FailClass::of(out).is_some());
3192 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194 if self
3198 .state
3199 .candidate_preservation
3200 .contains_key(&self.state.candidates[i].label)
3201 {
3202 self.state.seats.insert(seat.key.clone(), seat);
3203 }
3204 continue;
3205 }
3206 let seat_key = seat.key.clone();
3207 let agent = seat.agent.clone();
3216 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217 self.state.seats.insert(seat.key.clone(), seat);
3218 let label = self.state.candidates[i].label;
3219 self.state.interrupted_implementations.remove(&label);
3220
3221 let (summary, duration, failed, verified_claim) = match out {
3222 AgentOutcome::Ok(o) => {
3223 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224 let failed = (!o.usable()).then(|| {
3225 if o.timed_out {
3226 "agent timed out".to_owned()
3227 } else {
3228 format!(
3229 "agent exited with {}",
3230 crate::notices::exit_status(o.exit_code)
3231 )
3232 }
3233 });
3234 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235 (text, o.duration_ms, failed, verified_claim)
3236 }
3237 AgentOutcome::Dropped(o) => {
3243 let why = o
3244 .dropped
3245 .as_ref()
3246 .map(|d| d.why.as_str())
3247 .unwrap_or("the CLI ended the stream without delivering its answer");
3248 (
3249 String::new(),
3250 o.duration_ms,
3251 Some(format!("the CLI dropped the stream ({why})")),
3252 None,
3253 )
3254 }
3255 AgentOutcome::Quota(o) => {
3256 self.state.quota.push(QuotaLoss::from_report(
3257 seat_key,
3258 "implement",
3259 o.quota.as_ref(),
3260 ));
3261 (
3262 String::new(),
3263 o.duration_ms,
3264 Some("rate limited (quota); produced no change".to_owned()),
3265 None,
3266 )
3267 }
3268 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269 (String::new(), 0, Some(e), None)
3270 }
3271 };
3272
3273 let c = &mut self.state.candidates[i];
3276 if !exhausted_the_fallback_chain {
3277 c.agent = agent;
3278 }
3279 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280 c.duration_ms = duration;
3281 c.failed = failed;
3282 c.verified_noop = verified_claim;
3283 self.state
3284 .candidate_preservation
3285 .insert(label, "awaiting persistence".to_owned());
3286 }
3287 self.state.save()?;
3288 self.persist_candidates().await?;
3289
3290 if resource_blocked {
3291 return self.stop_for_resource();
3292 }
3293 if selected_failed {
3294 self.state.status = RunStatus::Blocked;
3295 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296 self.state.save()?;
3297 bail!(
3298 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299 );
3300 }
3301 self.after_implement()
3302 }
3303
3304 async fn persist_candidates(&mut self) -> Result<()> {
3306 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307 for label in labels {
3308 let i = self
3309 .state
3310 .candidates
3311 .iter()
3312 .position(|c| c.label == label)
3313 .context("pending candidate is missing")?;
3314 let c = self.state.candidates[i].clone();
3315 let interrupted = self.state.interrupted_implementations.contains(&label);
3316 let result = async {
3317 let wait = crate::repo_lock::WaitPolicy::new(
3318 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319 || false,
3320 );
3321 let _coordination =
3322 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323 if let Err(e) = git::snapshot_uncommitted(
3326 &c.worktree,
3327 &self
3328 .state
3329 .dir()
3330 .join("artifacts")
3331 .join(format!("candidate-{label}-recovery")),
3332 )
3333 .await
3334 {
3335 self.state.event(
3336 "preserve",
3337 format!("candidate {label} snapshot incomplete: {e:#}"),
3338 );
3339 }
3340 let rescued = git::rescue_commit(
3341 &c.worktree,
3342 &format!("magi: candidate {label} (uncommitted work)"),
3343 )
3344 .await?;
3345 self.state.note_withheld("implement", &rescued.withheld);
3346 let base = &self.state.base_commit;
3347 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352 if c.empty && commits > 0 && !patch.trim().is_empty()
3353 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355 || self.state.tally.is_some())
3356 {
3357 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361 }
3362 let c = &mut self.state.candidates[i];
3363 c.commits = commits;
3364 c.stat = stat;
3365 c.files = files;
3366 let no_changes = commits == 0 || patch.trim().is_empty();
3367 c.empty = no_changes && !interrupted;
3368 if interrupted && no_changes {
3369 c.failed = None;
3370 c.verified_noop = None;
3371 }
3372 if !c.empty {
3373 c.failed = None;
3374 c.verified_noop = None;
3375 }
3376 self.state.event(
3377 "preserve",
3378 format!("candidate {label}: {files} files, {commits} commits"),
3379 );
3380 Ok::<_, anyhow::Error>(())
3381 }
3382 .await;
3383 match result {
3384 Ok(()) => {
3385 self.state.candidate_preservation.remove(&label);
3386 if !interrupted || self.state.candidates[i].files > 0 {
3387 self.state.interrupted_implementations.remove(&label);
3388 }
3389 }
3390 Err(e) => {
3391 let reason = format!("candidate {label} preservation failed: {e:#}");
3392 self.state.event("preserve", &reason);
3393 self.state.candidate_preservation.insert(label, reason);
3394 }
3395 }
3396 if self.state.candidate_preservation.is_empty()
3397 && self.state.status == RunStatus::Blocked
3398 && self.state.candidates.iter().all(|c| c.failed.is_none())
3399 {
3400 self.state.status = RunStatus::Implementing;
3404 self.state.event(
3405 "preserve",
3406 "candidate persistence recovered; implementation can continue",
3407 );
3408 }
3409 self.state.save()?;
3410 }
3411 if !self.state.candidate_preservation.is_empty() {
3412 self.state.status = RunStatus::Blocked;
3413 self.state.save()?;
3414 bail!(
3415 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416 );
3417 }
3418 Ok(())
3419 }
3420
3421 async fn resume_undelivered(
3449 &mut self,
3450 results: &mut [(usize, SeatState, AgentOutcome)],
3451 sent: &[SeatJob],
3452 prompts: &Prompts,
3453 run_id: &str,
3454 ) {
3455 for (wi, seat, out) in results.iter_mut() {
3456 let Some(dropped) = (match &*out {
3457 AgentOutcome::Dropped(o) => o.dropped.clone(),
3458 _ => None,
3459 }) else {
3460 continue;
3461 };
3462 let Some(job) = sent.get(*wi) else { continue };
3463 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465 self.state.event(
3466 "implement",
3467 format!(
3468 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469 work is in the tree",
3470 seat.key, dropped.output_tokens, dropped.why
3471 ),
3472 );
3473 continue;
3474 }
3475 if !has_context(&job.spec, seat, job.sessions) {
3483 self.state.event(
3484 "implement",
3485 format!(
3486 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487 is no session left to resume",
3488 seat.key, dropped.output_tokens, dropped.why
3489 ),
3490 );
3491 continue;
3492 }
3493 self.state.event(
3494 "implement",
3495 format!(
3496 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497 conversation",
3498 seat.key, dropped.output_tokens, dropped.why
3499 ),
3500 );
3501 let mut retry = job.clone();
3502 retry.seat = seat.clone();
3503 retry.prompt = prompt::resume_after_drop(&dropped.why);
3504 retry.timeout = retry_budget(job.timeout, true);
3505 retry.stem = format!("{}-resume", job.stem);
3506 let cache = self.state.config.cache_dir();
3507 let ctx = WaveCtx {
3508 carry_seats: false,
3509 pauses: Some((&self.pause, &self.interrupt)),
3510 run: run_id,
3511 node: "implement",
3512 prompts,
3513 cache: cache.as_deref(),
3514 round: None,
3515 };
3516 let (resumed_seat, resumed) =
3517 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518 *seat = resumed_seat;
3519 *out = resumed;
3520 }
3521 }
3522
3523 async fn resume_seat_handovers(
3588 &mut self,
3589 results: &mut [(usize, SeatState, AgentOutcome)],
3590 sent: &mut [SeatJob],
3591 prompts: &Prompts,
3592 run_id: &str,
3593 ) -> Result<()> {
3594 let instruction = seeded_instruction(&self.state);
3595 let language = self.state.config.graph.language.clone();
3596 let github_language = self.state.config.graph.github_language.clone();
3597 let brief = self
3598 .state
3599 .advice
3600 .as_ref()
3601 .and_then(|a| a.synthesis.as_deref())
3602 .map(str::to_owned);
3603 let attachments = self.state.attachments.clone();
3604 for (wi, seat, out) in results.iter_mut() {
3605 let others: BTreeSet<String> = sent
3608 .iter()
3609 .enumerate()
3610 .filter(|(j, _)| j != wi)
3611 .map(|(_, j)| j.spec.id.clone())
3612 .collect();
3613 let Some(job) = sent.get_mut(*wi) else {
3614 continue;
3615 };
3616 let start = self
3621 .roles
3622 .implementer_roster
3623 .iter()
3624 .position(|s| s.id == job.spec.id)
3625 .unwrap_or(0);
3626 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627 let mut fallback_attempt = 0usize;
3628 let mut prev: Option<FailClass> = None;
3629 while let Some(cur) = FailClass::of(&*out) {
3630 if !should_hand_over(prev.as_ref(), &cur) {
3631 break;
3632 }
3633 let Some(next) =
3634 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635 .cloned()
3636 else {
3637 break;
3638 };
3639 tried.insert(next.id.clone());
3640 fallback_attempt += 1;
3641
3642 let coordination = self.coordinate(&job.cwd).await?;
3643 if let Ok(r) = git::rescue_commit(
3644 &job.cwd,
3645 &format!(
3646 "magi: candidate {} (uncommitted work before {} fallback)",
3647 seat.key,
3648 if cur == FailClass::Quota {
3649 "quota"
3650 } else {
3651 "handover"
3652 }
3653 ),
3654 )
3655 .await
3656 {
3657 self.state.note_withheld("implement", &r.withheld);
3658 }
3659 drop(coordination);
3660
3661 record_handover(
3662 &mut self.state,
3663 "implement",
3664 &seat.key,
3665 &seat.agent,
3666 &next.id,
3667 &cur,
3668 &fail_reason(&*out),
3669 );
3670 prev = Some(cur.clone());
3671
3672 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674 job.spec = next.clone();
3682 let mut retry = job.clone();
3683 retry.seat = new_seat;
3684 retry.prompt = prompt::implement_in(
3685 &instruction,
3686 &job.cwd.to_string_lossy(),
3687 &language,
3688 &github_language,
3689 brief.as_deref(),
3690 &attachments,
3691 );
3692 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693 let cache = self.state.config.cache_dir();
3694 let ctx = WaveCtx {
3695 carry_seats: false,
3696 pauses: Some((&self.pause, &self.interrupt)),
3697 run: run_id,
3698 node: "implement",
3699 prompts,
3700 cache: cache.as_deref(),
3701 round: None,
3702 };
3703 let (fallback_seat, fallback_out) = run_one(
3704 retry,
3705 Arc::clone(&self.sem),
3706 &ctx,
3707 &mut self.state,
3708 fallback_attempt,
3709 )
3710 .await;
3711 *seat = fallback_seat;
3712 *out = fallback_out;
3713 }
3714 }
3715 Ok(())
3716 }
3717
3718 async fn resume_unconfirmed_commands(
3742 &mut self,
3743 results: &mut [(usize, SeatState, AgentOutcome)],
3744 sent: &[SeatJob],
3745 prompts: &Prompts,
3746 run_id: &str,
3747 ) {
3748 for (wi, seat, out) in results.iter_mut() {
3749 let AgentOutcome::Ok(o) = &*out else {
3750 continue;
3751 };
3752 if !has_unconfirmed_command(&o.commands) {
3753 continue;
3754 }
3755 let Some(job) = sent.get(*wi) else { continue };
3756 if !has_context(&job.spec, seat, job.sessions) {
3757 self.state.event(
3758 "implement",
3759 format!(
3760 "{}: the reply named a command whose own CLI never confirmed the exit \
3761 status of, but there is no session left to resume",
3762 seat.key
3763 ),
3764 );
3765 continue;
3766 }
3767 self.state.event(
3768 "implement",
3769 format!(
3770 "{}: the reply named a command whose own CLI never confirmed the exit \
3771 status of; resuming the conversation",
3772 seat.key
3773 ),
3774 );
3775 let mut retry = job.clone();
3776 retry.seat = seat.clone();
3777 retry.prompt = prompt::resume_incomplete(
3778 "a command in your last reply had no confirmed exit status",
3779 );
3780 retry.timeout = retry_budget(job.timeout, true);
3781 retry.stem = format!("{}-confirm", job.stem);
3782 let cache = self.state.config.cache_dir();
3783 let ctx = WaveCtx {
3784 carry_seats: false,
3785 pauses: Some((&self.pause, &self.interrupt)),
3786 run: run_id,
3787 node: "implement",
3788 prompts,
3789 cache: cache.as_deref(),
3790 round: None,
3791 };
3792 let (resumed_seat, resumed) =
3793 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794 *seat = resumed_seat;
3795 *out = resumed;
3796 }
3797 }
3798
3799 async fn continue_fix_report(
3820 &mut self,
3821 mut seat: SeatState,
3822 parse_err: String,
3823 job: &SeatJob,
3824 prompts: &Prompts,
3825 run_id: &str,
3826 round: usize,
3827 ) -> (
3828 SeatState,
3829 Option<FixReport>,
3830 Option<String>,
3831 ContinuationRecord,
3832 ) {
3833 let mut last_err = parse_err;
3834 let mut cumulative_wait_ms = 0u64;
3835 let mut attempts = 0usize;
3836 loop {
3837 if !has_context(&job.spec, &seat, job.sessions) {
3838 self.state.event(
3839 "fix",
3840 format!(
3841 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842 session left to resume into"
3843 ),
3844 );
3845 let outcome = if attempts == 0 {
3846 ContinuationOutcome::NoSession
3847 } else {
3848 ContinuationOutcome::Exhausted
3849 };
3850 return (
3851 seat,
3852 None,
3853 Some(format!("unparsable fix report: {last_err}")),
3854 ContinuationRecord {
3855 attempts,
3856 cumulative_wait_ms,
3857 outcome,
3858 },
3859 );
3860 }
3861 if attempts >= MAX_FIX_CONTINUATIONS {
3862 self.state.event(
3863 "fix",
3864 format!(
3865 "round {round}: fixer's reply still had no adoption report after \
3866 {attempts} continuation(s) ({last_err}); giving up"
3867 ),
3868 );
3869 return (
3870 seat,
3871 None,
3872 Some(format!(
3873 "unparsable fix report after {attempts} continuation(s): {last_err}"
3874 )),
3875 ContinuationRecord {
3876 attempts,
3877 cumulative_wait_ms,
3878 outcome: ContinuationOutcome::Exhausted,
3879 },
3880 );
3881 }
3882 attempts += 1;
3883 self.state.event(
3884 "fix",
3885 format!(
3886 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888 ),
3889 );
3890 let mut retry = job.clone();
3891 retry.seat = seat.clone();
3892 retry.prompt = prompt::resume_incomplete(&last_err);
3893 retry.timeout = retry_budget(job.timeout, true);
3894 retry.stem = format!("{}-continue{attempts}", job.stem);
3895 let cache = self.state.config.cache_dir();
3896 let ctx = WaveCtx {
3897 carry_seats: false,
3898 pauses: Some((&self.pause, &self.interrupt)),
3899 run: run_id,
3900 node: "fix",
3901 prompts,
3902 cache: cache.as_deref(),
3903 round: Some(round),
3904 };
3905 let (resumed_seat, resumed_out) = run_one(
3906 retry,
3907 Arc::clone(&self.sem),
3908 &ctx,
3909 &mut self.state,
3910 attempts,
3911 )
3912 .await;
3913 seat = resumed_seat;
3914 match resumed_out {
3915 AgentOutcome::Ok(o) => {
3916 cumulative_wait_ms += o.duration_ms;
3917 match verdict::extract_json::<FixReport>(&o.text) {
3918 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919 self.state.event(
3920 "fix",
3921 format!(
3922 "round {round}: fixer's adoption report recovered after \
3923 {attempts} continuation(s)"
3924 ),
3925 );
3926 return (
3927 seat,
3928 Some(report),
3929 None,
3930 ContinuationRecord {
3931 attempts,
3932 cumulative_wait_ms,
3933 outcome: ContinuationOutcome::Resumed,
3934 },
3935 );
3936 }
3937 Ok(_) => {
3945 last_err = "the reply parsed, but it reported a command whose own CLI \
3946 never confirmed an exit status"
3947 .to_owned();
3948 }
3949 Err(e) => last_err = e.to_string(),
3950 }
3951 }
3952 AgentOutcome::Quota(o) => {
3953 cumulative_wait_ms += o.duration_ms;
3954 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956 self.state.quota.push(loss);
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: continuation rate limited (quota); not retrying now"
3961 ),
3962 );
3963 return (
3964 seat,
3965 None,
3966 Some("rate limited (quota) while recovering the fix report".to_owned()),
3967 ContinuationRecord {
3968 attempts,
3969 cumulative_wait_ms,
3970 outcome: ContinuationOutcome::QuotaLost,
3971 },
3972 );
3973 }
3974 AgentOutcome::Dropped(o) => {
3975 cumulative_wait_ms += o.duration_ms;
3976 let why = o
3977 .dropped
3978 .as_ref()
3979 .map(|d| d.why.as_str())
3980 .unwrap_or("the CLI ended the stream without delivering its answer");
3981 last_err = format!("the CLI dropped the stream ({why})");
3982 }
3983 AgentOutcome::Failed(e) => last_err = e,
3984 AgentOutcome::ResourceBlocked(e) => {
3987 return (
3988 seat,
3989 None,
3990 Some(format!("the continuation was not started: {e}")),
3991 ContinuationRecord {
3992 attempts: attempts.saturating_sub(1),
3993 cumulative_wait_ms,
3994 outcome: ContinuationOutcome::NoSession,
3995 },
3996 );
3997 }
3998 }
3999 }
4000 }
4001
4002 fn after_implement(&mut self) -> Result<()> {
4003 if !self.state.candidate_preservation.is_empty() {
4004 bail!("candidate preservation incomplete; judging is unsafe");
4005 }
4006 if self.state.leaks.is_empty() {
4008 let cfg = self.state.config.blind.clone();
4009 let mut leaks = Vec::new();
4010 for c in &self.state.candidates {
4011 let Some(patch) =
4012 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013 else {
4014 continue;
4015 };
4016 leaks.extend(blind::scan(
4017 &format!("candidate {} patch", c.label),
4018 &patch,
4019 &cfg.vendor_tokens,
4020 ));
4021 }
4022 if !leaks.is_empty() {
4023 let summary = leaks
4024 .iter()
4025 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026 .collect::<Vec<_>>()
4027 .join(", ");
4028 match cfg.on_leak {
4029 LeakPolicy::Fail => {
4030 self.state.status = RunStatus::Failed;
4031 self.state
4032 .event("blind", format!("vendor text in a patch: {summary}"));
4033 self.state.leaks = leaks;
4034 self.state.save()?;
4035 self.settle_questions();
4036 bail!(
4037 "blind.on_leak = \"fail\" and vendor text reached a \
4038 judged patch: {summary}"
4039 );
4040 }
4041 LeakPolicy::Redact => self.state.event(
4042 "blind",
4043 format!("redacting vendor text for judging: {summary}"),
4044 ),
4045 LeakPolicy::Warn => self.state.event(
4046 "blind",
4047 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048 ),
4049 }
4050 self.state.leaks = leaks;
4051 }
4052 }
4053
4054 if self.state.viable().is_empty() {
4055 if self.state.all_candidates_verified_noop() {
4056 self.state.status = RunStatus::VerifiedNoop;
4067 self.state.save()?;
4068 self.settle_questions();
4069 return Ok(());
4070 }
4071 self.state.status = RunStatus::Failed;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!("no candidate produced a change; nothing to judge");
4075 }
4076 self.state.status = RunStatus::Judging;
4077 self.state.save()?;
4078 Ok(())
4079 }
4080
4081 async fn judge(&mut self) -> Result<()> {
4084 let run_id = self.state.id.clone();
4089 let prompts = self.state.config.prompts.clone();
4090 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091 return Ok(());
4092 }
4093 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094 if viable.len() == 1 {
4095 self.state.judge_skipped = true;
4102 self.state.event(
4103 "judge",
4104 format!(
4105 "only candidate {} produced a change; judging skipped",
4106 viable[0].label
4107 ),
4108 );
4109 self.state.save()?;
4110 return Ok(());
4111 }
4112 self.state.status = RunStatus::Judging;
4113
4114 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115 let language = self.state.config.graph.language.clone();
4116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117 let sessions = self.state.config.graph.sessions;
4118 let artifacts = agent::artifacts_dir(&self.state.dir());
4119 let root = self.state.worktree_root();
4120 let base_short = short(&self.state.base_commit);
4121
4122 let mut jobs = Vec::new();
4123 let mut orders = Vec::new();
4124 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128 let seat_key = format!("judge-{}", j + 1);
4129 let seat = self.seat(&seat_key, &spec.id);
4130 jobs.push(SeatJob {
4131 prompt: prompt::judge(
4132 &self.state.instruction,
4133 &views,
4134 self.roles.judges.len(),
4135 &base_short,
4136 &language,
4137 ),
4138 spec,
4139 seat,
4140 cwd: root.join(format!("judge-{}", j + 1)),
4141 timeout,
4142 allow_write: false,
4143 sessions,
4144 artifacts: artifacts.clone(),
4145 stem: format!("judge-{}", j + 1),
4146 handover: None,
4147 });
4148 }
4149
4150 self.state.event(
4151 "judge",
4152 format!(
4153 "{} judges ranking {} candidates blind",
4154 jobs.len(),
4155 viable.len()
4156 ),
4157 );
4158 let labels_for_check = labels.clone();
4159 let mut quota_losses = Vec::new();
4160 let cache = self.state.config.cache_dir();
4161 let ctx = WaveCtx {
4162 carry_seats: false,
4163 pauses: Some((&self.pause, &self.interrupt)),
4164 run: &run_id,
4165 node: "judge",
4166 prompts: &prompts,
4167 cache: cache.as_deref(),
4168 round: None,
4169 };
4170 let results = ask_json_wave::<Ranking>(
4171 jobs,
4172 Arc::clone(&self.sem),
4173 self.state.config.graph.retries,
4174 &self.roles.judge_roster,
4175 &ctx,
4176 &mut quota_losses,
4177 &mut self.state,
4178 &move |r: &Ranking| r.validate(&labels_for_check),
4179 )
4180 .await;
4181 if self.state.resource_block.is_some() {
4182 return self.stop_for_resource();
4183 }
4184 self.state.quota.extend(quota_losses);
4185
4186 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187 let agent_id = seat.agent.clone();
4188 self.state.seats.insert(seat.key.clone(), seat);
4189 let mut record = Judgement {
4190 judge: j + 1,
4191 seat: format!("judge-{}", j + 1),
4192 agent: agent_id,
4193 ranking: Vec::new(),
4194 reasons: BTreeMap::new(),
4195 confidence: None,
4196 strengths: BTreeMap::new(),
4197 order: orders[j].clone(),
4198 failed: None,
4199 duration_ms: 0,
4200 };
4201 match res {
4202 Ok((ranking, out)) => {
4203 record.ranking = ranking.normalized();
4204 record.reasons = ranking.reasons;
4205 record.strengths = ranking.strengths;
4206 record.confidence = ranking.confidence;
4207 record.duration_ms = out.duration_ms;
4208 self.state.event(
4209 "judge",
4210 format!(
4211 "judge {} ranked {}",
4212 j + 1,
4213 record.ranking.iter().collect::<String>()
4214 ),
4215 );
4216 }
4217 Err(e) => {
4218 record.failed = Some(e.to_string());
4219 self.state
4220 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221 }
4222 }
4223 self.state.judgements.push(record);
4224 self.state.save()?;
4225 }
4226 Ok(())
4227 }
4228
4229 async fn deliberate(&mut self) -> Result<()> {
4232 if self.hold_base_sync().await? {
4233 return Ok(());
4234 }
4235 let run_id = self.state.id.clone();
4240 let prompts = self.state.config.prompts.clone();
4241 let rounds = self.state.config.graph.deliberate_rounds;
4242 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244 return Ok(());
4245 }
4246 let tops: Vec<char> = self
4247 .state
4248 .judgements
4249 .iter()
4250 .filter_map(|j| j.ranking.first().copied())
4251 .collect();
4252 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254 self.state.event(
4255 "deliberate",
4256 format!("judges agreed on {} outright; no deliberation", tops[0]),
4257 );
4258 }
4259 self.state.status = RunStatus::Voting;
4260 self.state.save()?;
4261 return Ok(());
4262 }
4263
4264 self.state.status = RunStatus::Deliberating;
4265 self.state.event(
4266 "deliberate",
4267 format!(
4268 "split: first choices were {} — opening {rounds} round(s)",
4269 tops.iter().collect::<String>()
4270 ),
4271 );
4272
4273 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274 let language = self.state.config.graph.language.clone();
4275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276 let sessions = self.state.config.graph.sessions;
4277 let artifacts = agent::artifacts_dir(&self.state.dir());
4278 let root = self.state.worktree_root();
4279 let base_short = short(&self.state.base_commit);
4280
4281 for round in self.state.deliberation.len() + 1..=rounds {
4285 let mut turns: Vec<DeliberationTurn> = Vec::new();
4286 let mut round_losses = Vec::new();
4287 let mut asked = 0;
4288 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289 if self.state.judgements[j].failed.is_some() {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let mut seat = self.seat(&seat_key, &spec.id);
4295 let transcript = self.transcript(&turns, j);
4296 let build = |context: Option<&str>| {
4297 prompt::deliberate(
4298 &self.state.instruction,
4299 context,
4300 &transcript,
4301 round,
4302 rounds,
4303 &language,
4304 )
4305 };
4306 let block = self.candidate_block(&viable, &base_short);
4307 let full = build(Some(&block));
4308 let text = if has_context(&spec, &seat, sessions) {
4309 build(None)
4310 } else {
4311 full.clone()
4312 };
4313 let job = SeatJob {
4314 spec,
4315 seat: seat.clone(),
4316 prompt: text,
4317 cwd: root.join(format!("judge-{}", j + 1)),
4318 timeout,
4319 allow_write: false,
4320 sessions,
4321 artifacts: artifacts.clone(),
4322 stem: format!("delib-{round}-judge-{}", j + 1),
4323 handover: Some(full),
4324 };
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "deliberate",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let mut losses = Vec::new();
4339 asked += 1;
4340 let mut results = ask_wave_with::<String>(
4341 vec![job],
4342 Arc::clone(&self.sem),
4343 0,
4344 &self.roles.judge_roster,
4345 &ctx,
4346 &mut losses,
4347 &mut self.state,
4348 &|text: &str| {
4349 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 return self.stop_for_resource();
4355 }
4356 round_losses.extend(losses.iter().cloned());
4357 self.state.quota.extend(losses);
4358 let (updated, res, _) = results.pop().expect("one job in, one result out");
4359 seat = updated;
4360 let agent_id = seat.agent.clone();
4361 self.state.seats.insert(seat.key.clone(), seat);
4362 let body = match res {
4363 Ok((body, _)) => body,
4364 Err(e) => {
4367 self.state
4368 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369 continue;
4370 }
4371 };
4372 let tentative = verdict::extract_json::<Position>(&body)
4373 .ok()
4374 .and_then(|p| p.tentative)
4375 .and_then(|s| s.trim().chars().next())
4376 .map(|c| c.to_ascii_uppercase());
4377 self.state.event(
4378 "deliberate",
4379 format!(
4380 "round {round}: judge {} now favours {}",
4381 j + 1,
4382 tentative.map_or("—".to_owned(), |c| c.to_string())
4383 ),
4384 );
4385 turns.push(DeliberationTurn {
4386 judge: j + 1,
4387 agent: agent_id,
4388 body: blind::sanitize_prose(&body, &self.state.config.blind),
4389 tentative,
4390 });
4391 }
4392 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394 return self.stop_for_resource();
4395 }
4396 self.state
4397 .deliberation
4398 .push(DeliberationRound { round, turns });
4399 self.state.save()?;
4400 }
4401
4402 self.state.status = RunStatus::Voting;
4403 self.state.save()?;
4404 Ok(())
4405 }
4406
4407 async fn vote(&mut self) -> Result<()> {
4410 if self.hold_base_sync().await? {
4411 return Ok(());
4412 }
4413 let run_id = self.state.id.clone();
4418 let prompts = self.state.config.prompts.clone();
4419 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420 return Ok(());
4421 }
4422 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423 if viable.len() == 1 {
4424 return Ok(());
4425 }
4426 self.state.status = RunStatus::Voting;
4427
4428 let language = self.state.config.graph.language.clone();
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let root = self.state.worktree_root();
4433 let base_short = short(&self.state.base_commit);
4434 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436 let mut jobs = Vec::new();
4437 let mut seats_at = Vec::new();
4438 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439 if self
4440 .state
4441 .judgements
4442 .get(j)
4443 .is_some_and(|r| r.failed.is_some())
4444 {
4445 continue;
4446 }
4447 let seat_key = format!("judge-{}", j + 1);
4448 let spec = self.occupant(&seat_key, spec);
4449 let seat = self.seat(&seat_key, &spec.id);
4450 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451 let text = if has_context(&spec, &seat, sessions) {
4452 prompt::final_vote(&viable, &language)
4453 } else {
4454 full.clone()
4455 };
4456 jobs.push(SeatJob {
4457 spec,
4458 seat,
4459 prompt: text,
4460 cwd: root.join(format!("judge-{}", j + 1)),
4461 timeout,
4462 allow_write: false,
4463 sessions,
4464 artifacts: artifacts.clone(),
4465 stem: format!("vote-judge-{}", j + 1),
4466 handover: Some(full),
4467 });
4468 seats_at.push(j);
4469 }
4470
4471 self.state.event(
4472 "vote",
4473 format!(
4474 "collecting {} final votes one by one, privately",
4475 jobs.len()
4476 ),
4477 );
4478 let allowed = viable.clone();
4479 let mut quota_losses = Vec::new();
4480 let cache = self.state.config.cache_dir();
4481 let ctx = WaveCtx {
4482 carry_seats: false,
4483 pauses: Some((&self.pause, &self.interrupt)),
4484 run: &run_id,
4485 node: "vote",
4486 prompts: &prompts,
4487 cache: cache.as_deref(),
4488 round: None,
4489 };
4490 let results = ask_json_wave::<FinalVote>(
4491 jobs,
4492 Arc::clone(&self.sem),
4493 self.state.config.graph.retries,
4494 &self.roles.judge_roster,
4495 &ctx,
4496 &mut quota_losses,
4497 &mut self.state,
4498 &move |v: &FinalVote| match v.label() {
4499 Some(c) if allowed.contains(&c) => Ok(()),
4500 other => bail!("vote {other:?} is not one of {allowed:?}"),
4501 },
4502 )
4503 .await;
4504 if self.state.resource_block.is_some() {
4505 return self.stop_for_resource();
4506 }
4507 self.state.quota.extend(quota_losses);
4508
4509 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510 let agent_id = seat.agent.clone();
4511 self.state.seats.insert(seat.key.clone(), seat);
4512 let initial = self
4513 .state
4514 .judgements
4515 .get(j)
4516 .and_then(|r| r.ranking.first().copied());
4517 let mut record = VoteRecord {
4518 judge: j + 1,
4519 agent: agent_id,
4520 vote: None,
4521 reason: String::new(),
4522 changed: false,
4523 };
4524 match res {
4525 Ok((v, _)) => {
4526 record.vote = v.label();
4527 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529 self.state.event(
4530 "vote",
4531 format!(
4532 "judge {} voted {}{}",
4533 j + 1,
4534 record.vote.unwrap_or('?'),
4535 if record.changed { " (changed)" } else { "" }
4536 ),
4537 );
4538 }
4539 Err(e) => {
4540 self.state
4541 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542 }
4543 }
4544 self.state.votes.push(record);
4545 self.state.save()?;
4546 }
4547 Ok(())
4548 }
4549
4550 fn tally(&mut self) -> Result<()> {
4553 if self.state.tally.is_some() {
4554 return Ok(());
4555 }
4556 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557 let tops: Vec<char> = self
4558 .state
4559 .judgements
4560 .iter()
4561 .filter_map(|j| j.ranking.first().copied())
4562 .collect();
4563 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568 let mut cast: Vec<char> = Vec::new();
4569 for (i, j) in self.state.judgements.iter().enumerate() {
4570 let vote = self
4571 .state
4572 .votes
4573 .iter()
4574 .find(|v| v.judge == i + 1)
4575 .and_then(|v| v.vote)
4576 .or_else(|| j.ranking.first().copied());
4577 if let Some(v) = vote {
4578 *first_choice.entry(v).or_insert(0) += 1;
4579 cast.push(v);
4580 }
4581 }
4582
4583 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584 for j in &self.state.judgements {
4585 let n = j.ranking.len();
4586 for (pos, label) in j.ranking.iter().enumerate() {
4587 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588 }
4589 }
4590
4591 let best = first_choice.values().copied().max().unwrap_or(0);
4592 let mut leaders: Vec<char> = first_choice
4593 .iter()
4594 .filter(|(_, v)| **v == best)
4595 .map(|(k, _)| *k)
4596 .collect();
4597 let mut tie_break = None;
4598 if leaders.len() > 1 {
4599 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600 let borda_leaders: Vec<char> = leaders
4601 .iter()
4602 .copied()
4603 .filter(|l| borda[l] == top_borda)
4604 .collect();
4605 tie_break = Some(if borda_leaders.len() == 1 {
4606 format!(
4607 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608 leaders.len()
4609 )
4610 } else {
4611 format!(
4612 "{} way tie on both first-choice votes and Borda points, broken by label order",
4613 leaders.len()
4614 )
4615 });
4616 leaders = borda_leaders;
4617 leaders.sort_unstable();
4618 }
4619 let winner = *leaders
4620 .first()
4621 .or(viable.first())
4622 .context("no candidate to declare a winner from")?;
4623
4624 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626 let deliberated = !self.state.deliberation.is_empty();
4627
4628 let quota_seats: std::collections::BTreeSet<&str> =
4632 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633 let mut present = 0usize;
4634 for (i, j) in self.state.judgements.iter().enumerate() {
4635 if quota_seats.contains(j.seat.as_str()) {
4636 continue;
4637 }
4638 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639 let voted = self
4640 .state
4641 .votes
4642 .iter()
4643 .any(|v| v.judge == i + 1 && v.vote.is_some());
4644 if ranked || voted {
4645 present += 1;
4646 }
4647 }
4648 let needs_quorum = viable.len() > 1;
4654 let judges_total = if needs_quorum {
4655 self.roles.judges.len()
4656 } else {
4657 0
4658 };
4659 let quorum = if needs_quorum {
4660 judges_total / 2 + 1
4661 } else {
4662 0
4663 };
4664 let met_quorum = !needs_quorum || present >= quorum;
4665 let uncontested = (!needs_quorum).then(|| {
4666 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667 });
4668
4669 self.state.event(
4670 "tally",
4671 match &uncontested {
4672 Some(reason) => format!("winner {winner} — {reason}"),
4673 None => format!(
4674 "winner {winner} — votes {} | initial {} | {} changed | \
4675 {present}/{judges_total} judges{}",
4676 first_choice
4677 .iter()
4678 .map(|(k, v)| format!("{k}:{v}"))
4679 .collect::<Vec<_>>()
4680 .join(" "),
4681 if unanimous_initial {
4682 "unanimous"
4683 } else {
4684 "split"
4685 },
4686 changed_votes,
4687 if met_quorum {
4688 String::new()
4689 } else {
4690 format!(" — below quorum ({quorum} required)")
4691 },
4692 ),
4693 },
4694 );
4695 if !met_quorum {
4696 self.state.event(
4697 "stall",
4698 format!(
4699 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700 the run stops here, resumable"
4701 ),
4702 );
4703 }
4704 self.state.tally = Some(Tally {
4705 first_choice,
4706 borda,
4707 winner,
4708 rankings: tops.len(),
4709 unanimous_initial,
4710 deliberated,
4711 changed_votes,
4712 unanimous_final,
4713 tie_break,
4714 judges: judges_total,
4715 present,
4716 quorum,
4717 met_quorum,
4718 uncontested,
4719 });
4720 self.state.status = if met_quorum {
4721 RunStatus::Reviewing
4722 } else {
4723 RunStatus::Stalled
4724 };
4725 self.state.save()?;
4726 Ok(())
4727 }
4728
4729 #[allow(clippy::too_many_lines)]
4750 async fn recover_stall(&mut self) -> Result<bool> {
4751 let run_id = self.state.id.clone();
4756 let prompts = self.state.config.prompts.clone();
4757 let quota_seats: BTreeSet<&str> =
4762 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763 let absent: Vec<String> = self
4764 .state
4765 .judgements
4766 .iter()
4767 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768 .map(|j| j.seat.clone())
4769 .collect();
4770 if absent.is_empty() {
4771 return Ok(false);
4772 }
4773 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774 if viable.len() <= 1 {
4775 return Ok(false);
4776 }
4777 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778 let language = self.state.config.graph.language.clone();
4779 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780 let sessions = self.state.config.graph.sessions;
4781 let artifacts = agent::artifacts_dir(&self.state.dir());
4782 let root = self.state.worktree_root();
4783 let base_short = short(&self.state.base_commit);
4784 let candidates: Vec<Candidate> = viable.clone();
4785
4786 let mut positions: Vec<usize> = absent
4788 .iter()
4789 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790 .collect();
4791 if positions.is_empty() {
4792 return Ok(false);
4793 }
4794 positions.sort_unstable();
4795 positions.dedup();
4796
4797 let mut judge_jobs = Vec::new();
4799 for &j in &positions {
4800 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802 let seat_key = format!("judge-{}", j + 1);
4803 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804 let seat = self.seat(&seat_key, &spec.id);
4805 judge_jobs.push(SeatJob {
4806 spec,
4807 seat,
4808 prompt: prompt::judge(
4809 &self.state.instruction,
4810 &views,
4811 self.roles.judges.len(),
4812 &base_short,
4813 &language,
4814 ),
4815 cwd: root.join(seat_key),
4816 timeout,
4817 allow_write: false,
4818 sessions,
4819 artifacts: artifacts.clone(),
4820 stem: format!("judge-{}-recover", j + 1),
4821 handover: None,
4822 });
4823 }
4824
4825 let labels_for_check = labels.clone();
4826 let mut judge_losses = Vec::new();
4827 let retries = self.state.config.graph.retries;
4828 let cache = self.state.config.cache_dir();
4829 let ctx = WaveCtx {
4830 carry_seats: false,
4831 pauses: Some((&self.pause, &self.interrupt)),
4832 run: &run_id,
4833 node: "judge",
4834 prompts: &prompts,
4835 cache: cache.as_deref(),
4836 round: None,
4837 };
4838 let results = ask_json_wave::<Ranking>(
4839 judge_jobs,
4840 Arc::clone(&self.sem),
4841 retries,
4842 &self.roles.judge_roster,
4843 &ctx,
4844 &mut judge_losses,
4845 &mut self.state,
4846 &move |r: &Ranking| r.validate(&labels_for_check),
4847 )
4848 .await;
4849 if self.state.resource_block.is_some() {
4850 self.stop_for_resource()?;
4851 return Ok(false);
4852 }
4853
4854 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857 let agent_id = seat.agent.clone();
4858 self.state.seats.insert(seat.key.clone(), seat);
4859 let record = &mut self.state.judgements[j];
4860 match res {
4861 Ok((ranking, out)) => {
4862 record.agent = agent_id;
4863 record.ranking = ranking.normalized();
4864 record.reasons = ranking.reasons;
4865 record.strengths = ranking.strengths;
4866 record.confidence = ranking.confidence;
4867 record.failed = None;
4868 record.duration_ms = out.duration_ms;
4869 recovered.insert(j);
4870 self.state.event(
4871 "recover",
4872 format!("judge {} ranked again after the limit", j + 1),
4873 );
4874 }
4875 Err(e) => {
4876 self.state
4877 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878 }
4879 }
4880 }
4881
4882 let mut vote_jobs = Vec::new();
4884 let mut vote_pos: Vec<usize> = Vec::new();
4885 for &j in &recovered {
4886 let seat_key = format!("judge-{}", j + 1);
4887 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888 let seat = self.seat(&seat_key, &spec.id);
4889 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890 let text = if has_context(&spec, &seat, sessions) {
4891 prompt::final_vote(&labels, &language)
4892 } else {
4893 full.clone()
4894 };
4895 vote_jobs.push(SeatJob {
4896 spec,
4897 seat,
4898 prompt: text,
4899 cwd: root.join(seat_key),
4900 timeout,
4901 allow_write: false,
4902 sessions,
4903 artifacts: artifacts.clone(),
4904 stem: format!("vote-judge-{}-recover", j + 1),
4905 handover: Some(full),
4906 });
4907 vote_pos.push(j);
4908 }
4909 let allowed = labels.clone();
4910 let mut vote_losses = Vec::new();
4911 let vote_retries = self.state.config.graph.retries;
4912 let vote_cache = self.state.config.cache_dir();
4913 let ctx = WaveCtx {
4914 carry_seats: false,
4915 pauses: Some((&self.pause, &self.interrupt)),
4916 run: &run_id,
4917 node: "vote",
4918 prompts: &prompts,
4919 cache: vote_cache.as_deref(),
4920 round: None,
4921 };
4922 let votes = ask_json_wave::<FinalVote>(
4923 vote_jobs,
4924 Arc::clone(&self.sem),
4925 vote_retries,
4926 &self.roles.judge_roster,
4927 &ctx,
4928 &mut vote_losses,
4929 &mut self.state,
4930 &move |v: &FinalVote| match v.label() {
4931 Some(c) if allowed.contains(&c) => Ok(()),
4932 other => bail!("vote {other:?} is not one of {allowed:?}"),
4933 },
4934 )
4935 .await;
4936 if self.state.resource_block.is_some() {
4937 self.stop_for_resource()?;
4938 return Ok(false);
4939 }
4940 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941 let agent_id = seat.agent.clone();
4942 self.state.seats.insert(seat.key.clone(), seat);
4943 match res {
4944 Ok((v, _)) => {
4945 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946 rec.vote = v.label();
4947 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948 } else {
4949 self.state.votes.push(VoteRecord {
4950 judge: j + 1,
4951 agent: agent_id,
4952 vote: v.label(),
4953 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954 changed: false,
4955 });
4956 }
4957 self.state.event(
4958 "recover",
4959 format!("judge {} voted again after the limit", j + 1),
4960 );
4961 }
4962 Err(e) => {
4963 self.state
4964 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965 }
4966 }
4967 }
4968
4969 let recovered_keys: BTreeSet<String> = recovered
4973 .iter()
4974 .map(|&j| format!("judge-{}", j + 1))
4975 .collect();
4976 self.state
4977 .quota
4978 .retain(|q| !recovered_keys.contains(&q.seat));
4979 for loss in judge_losses.into_iter().chain(vote_losses) {
4983 if recovered_keys.contains(&loss.seat) {
4984 continue;
4985 }
4986 self.state.quota.retain(|q| q.seat != loss.seat);
4987 self.state.quota.push(loss);
4988 }
4989
4990 self.state.tally = None;
4992 self.tally()?;
4993 Ok(self
4994 .state
4995 .tally
4996 .as_ref()
4997 .map(|t| t.met_quorum)
4998 .unwrap_or(false))
4999 }
5000
5001 async fn graft(&mut self) -> Result<()> {
5017 let Err(e) = self.graft_inner().await else {
5018 return Ok(());
5019 };
5020 if e.is::<crate::repo_lock::ResourceBlocked>() {
5021 return Err(e);
5022 }
5023 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {e:#}"));
5026 self.state.save()?;
5027 return Err(e);
5028 }
5029 if let Some(rec) = self.state.graft.clone()
5034 && !rec.outcome.settled()
5035 {
5036 if let Some(winner) = self.state.winner().cloned() {
5037 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038 if re.is::<crate::repo_lock::ResourceBlocked>() {
5039 return Err(re);
5040 }
5041 self.state.status = RunStatus::Blocked;
5042 self.state.event("graft", format!("blocked: {re:#}"));
5043 self.state.save()?;
5044 return Err(re);
5045 }
5046 }
5047 if let Some(g) = self.state.graft.as_mut() {
5048 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049 }
5050 }
5051 self.state.event("graft", format!("skipped: {e:#}"));
5052 if let Err(e) = self.state.save() {
5053 tracing::warn!("save after graft: {e:#}");
5054 }
5055 Ok(())
5056 }
5057
5058 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061 let _coordination = self.coordinate(wt).await?;
5062 crate::recovery::restore_repair(wt, pre)
5063 .await
5064 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065 }
5066
5067 fn graft_skipped(&mut self, why: &str) {
5069 let message = format!("skipped: {why}");
5070 if !self
5071 .state
5072 .events
5073 .iter()
5074 .any(|e| e.node == "graft" && e.message == message)
5075 {
5076 self.state.event("graft", message);
5077 }
5078 }
5079
5080 async fn graft_inner(&mut self) -> Result<()> {
5081 if let Some(rec) = self.state.graft.clone()
5084 && !rec.outcome.settled()
5085 {
5086 if self.state.reviews.is_empty()
5090 && let Some(winner) = self.state.winner().cloned()
5091 {
5092 self.graft_restore(&winner.worktree, &rec.pre).await?;
5093 }
5094 if rec.outcome == GraftOutcome::WaitingForRestore {
5095 self.state.graft = None;
5096 self.state.save()?;
5097 } else {
5098 if let Some(g) = self.state.graft.as_mut() {
5099 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100 }
5101 self.state.event(
5102 "graft",
5103 "rolled back: the previous round was interrupted; the winner is untouched",
5104 );
5105 self.state.save()?;
5106 return Ok(());
5107 }
5108 }
5109 let plan = match graft::plan(&self.state) {
5110 Ok(plan) => plan,
5111 Err(skip) => {
5112 if skip != graft::Skip::AlreadyDone {
5113 self.graft_skipped(skip.reason());
5114 self.state.save()?;
5115 }
5116 return Ok(());
5117 }
5118 };
5119 let Some(winner) = self.state.winner().cloned() else {
5120 return Ok(());
5121 };
5122 let wt = winner.worktree.clone();
5123 if !git::is_clean(&wt).await? {
5124 self.graft_skipped("the winner's worktree has uncommitted changes");
5125 self.state.save()?;
5126 return Ok(());
5127 }
5128 let pre = git::rev_parse(&wt, "HEAD").await?;
5129 self.state.graft = Some(GraftRecord {
5132 pre: pre.clone(),
5133 post: None,
5134 lines: None,
5135 limit: None,
5136 agent: None,
5137 outcome: GraftOutcome::Running,
5138 requested: graft::requested(&plan),
5139 adoption: None,
5140 adoption_error: None,
5141 });
5142 self.state.event(
5143 "graft",
5144 format!(
5145 "porting strengths of candidate(s) {} onto {}",
5146 plan.strengths
5147 .iter()
5148 .map(|(l, _, _)| l.to_string())
5149 .collect::<Vec<_>>()
5150 .join(", "),
5151 plan.winner
5152 ),
5153 );
5154 self.state.save()?;
5155
5156 let fix_prompt = prompt::graft(
5157 &self.state.instruction,
5158 plan.winner,
5159 &plan.strengths,
5160 &self.state.config.graph.language,
5161 );
5162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163 let sessions = self.state.config.graph.sessions;
5164 let artifacts = agent::artifacts_dir(&self.state.dir());
5165 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166 let (_, seat, out) = self
5167 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168 SeatJob {
5169 prompt: fix_prompt.clone(),
5170 spec,
5171 seat,
5172 cwd: wt.clone(),
5173 timeout,
5174 allow_write: true,
5175 sessions,
5176 artifacts: artifacts.clone(),
5177 stem: "graft".to_owned(),
5178 handover: None,
5179 }
5180 })
5181 .await;
5182 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183 self.state.graft.as_mut().expect("graft record").outcome =
5186 GraftOutcome::WaitingForRestore;
5187 return self.stop_for_resource();
5188 }
5189 let mut why: Option<String> = None;
5190 match &out {
5191 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192 why = Some("the fixer produced nothing usable".to_owned());
5193 }
5194 AgentOutcome::Ok(_) => {}
5195 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196 AgentOutcome::Quota(o) => {
5197 self.state.quota.push(QuotaLoss::from_report(
5198 seat.key.clone(),
5199 "graft",
5200 o.quota.as_ref(),
5201 ));
5202 why = Some("rate limited (quota)".to_owned());
5203 }
5204 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205 why = Some(format!(
5206 "the fixer failed: {}",
5207 e.lines()
5208 .next()
5209 .unwrap_or("")
5210 .chars()
5211 .take(160)
5212 .collect::<String>()
5213 ));
5214 }
5215 }
5216 let agent_id = seat.agent.clone();
5217 self.state.seats.insert(seat.key.clone(), seat);
5218
5219 let mut post = None;
5220 let mut lines = None;
5221 let _coordination = self.coordinate(&wt).await?;
5222 if why.is_none() {
5223 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224 if let Ok(r) = &rescued {
5225 self.state.note_withheld("graft", &r.withheld);
5226 }
5227 let head = git::rev_parse(&wt, "HEAD").await?;
5228 let dirty = match &rescued {
5232 Ok(r) => git::status_porcelain(&wt)
5233 .await
5234 .map(|st| {
5235 st.lines().any(|l| {
5236 !r.withheld
5237 .iter()
5238 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239 })
5240 })
5241 .unwrap_or(true),
5242 Err(_) => true,
5243 };
5244 if dirty {
5245 why = Some("the fixer left changes that could not be committed".to_owned());
5246 } else if head == pre {
5247 why = Some("the fixer changed nothing".to_owned());
5248 } else {
5249 lines = git::changed_lines(&wt, &pre, &head).await;
5250 post = Some(head);
5251 }
5252 }
5253 let (adoption, adoption_error) = match &out {
5254 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255 _ => (None, None),
5256 };
5257 let requested = graft::requested(&plan);
5258 let outcome = match why {
5259 None => {
5260 self.state.event(
5261 "graft",
5262 format!(
5263 "applied: {}",
5264 lines.map_or_else(
5265 || "changed lines unavailable".to_owned(),
5266 |n| format!("{n} changed lines")
5267 )
5268 ),
5269 );
5270 GraftOutcome::Applied
5271 }
5272 Some(why) => {
5273 crate::recovery::restore_repair(&wt, &pre)
5276 .await
5277 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278 self.state.event(
5279 "graft",
5280 format!("rolled back: {why}; the winner is untouched"),
5281 );
5282 GraftOutcome::RolledBack(why)
5283 }
5284 };
5285 self.state.graft = Some(GraftRecord {
5286 pre,
5287 post,
5288 lines,
5289 limit: None,
5290 agent: Some(agent_id),
5291 outcome,
5292 requested,
5293 adoption,
5294 adoption_error,
5295 });
5296 self.state.save()?;
5297 Ok(())
5298 }
5299
5300 async fn fold_losers(&mut self) -> Result<()> {
5303 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304 return Ok(());
5305 };
5306 if !self.state.candidate_preservation.is_empty() {
5307 bail!("candidate preservation incomplete; refusing cleanup");
5308 }
5309 let repo = self.state.repo.clone();
5310 let _coordination = self.coordinate(&repo).await?;
5311 let mut folded = Vec::new();
5312 for i in 0..self.state.candidates.len() {
5313 let c = &self.state.candidates[i];
5314 if c.label == winner || c.folded {
5315 continue;
5316 }
5317 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318 remove_preserved_worktree(&repo, &wt).await?;
5319 if git::branch_exists(&repo, &branch).await?
5320 && !git::branch_delete(&repo, &branch).await?
5321 {
5322 bail!("candidate branch removal refused");
5323 }
5324 self.state.candidates[i].folded = true;
5325 self.state.save()?;
5326 folded.push(label.to_string());
5327 }
5328 let root = self.state.worktree_root();
5330 for j in 1..=self.roles.judges.len() {
5331 let wt = root.join(format!("judge-{j}"));
5332 if wt.exists() {
5333 remove_preserved_worktree(&repo, &wt).await?;
5334 }
5335 }
5336 if self.state.config.graph.advise {
5339 for k in 1..=self.state.config.graph.advisors {
5340 let wt = root.join(format!("advisor-{k}"));
5341 if wt.exists() {
5342 remove_preserved_worktree(&repo, &wt).await?;
5343 }
5344 }
5345 }
5346 if !folded.is_empty() {
5347 self.state
5348 .event("fold", format!("folded candidates {}", folded.join(", ")));
5349 self.state.save()?;
5350 }
5351 Ok(())
5352 }
5353
5354 async fn hold_base_sync(&mut self) -> Result<bool> {
5357 self.hold_base_sync_as(None).await
5358 }
5359
5360 async fn hold_base_sync_kind(
5361 &mut self,
5362 kind: crate::base_sync_recovery::StopKind,
5363 ) -> Result<()> {
5364 self.hold_base_sync_as(Some(kind)).await?;
5365 Ok(())
5366 }
5367
5368 async fn hold_base_sync_as(
5369 &mut self,
5370 kind: Option<crate::base_sync_recovery::StopKind>,
5371 ) -> Result<bool> {
5372 if !self.state.base_sync_blocked() {
5373 self.state.base_sync_recovery.stop = None;
5374 return Ok(false);
5375 }
5376 let head = match self.state.winner() {
5377 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5378 .await
5379 .unwrap_or_default(),
5380 None => String::new(),
5381 };
5382 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5383 && !authorization.consumed
5384 && authorization.stop.head != head
5385 {
5386 authorization.refuse(format!(
5387 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5388 authorization.stop.head
5389 ));
5390 }
5391 self.state.record_base_sync_stop(&head);
5392 if let Some(kind) = kind {
5393 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5394 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5395 stop.kind = kind;
5396 }
5397 }
5398 self.state.save()?;
5399 Ok(true)
5400 }
5401
5402 async fn sync_to_base(&mut self) -> Result<()> {
5444 if self.state.status == RunStatus::AlreadyInBase {
5445 return Ok(());
5446 }
5447 self.hold_base_sync().await?;
5448 let conflicted = self
5449 .state
5450 .base_sync
5451 .as_ref()
5452 .is_some_and(|s| s.conflict.is_some());
5453 let Some(winner) = self.state.winner().cloned() else {
5454 return Ok(());
5455 };
5456
5457 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5458
5459 let repo = self.state.repo.clone();
5460 let remote = self.state.config.merge.remote.clone();
5461 let base_branch = self.state.base_branch.clone();
5462 let tracking = format!("{remote}/{base_branch}");
5463
5464 let fetched_base =
5465 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5466 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5471 return Ok(());
5472 };
5473
5474 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5475 self.state.ensure_base_episode();
5476 let mut pending = self
5477 .state
5478 .base_sync_recovery
5479 .operations
5480 .last()
5481 .filter(|o| !o.finished)
5482 .cloned();
5483 let scratch = self.state.dir().join("base-sync");
5484 if git::rebase_in_progress(&scratch).await {
5485 if pending.is_none()
5488 && self.state.base_sync_recovery.operations.is_empty()
5489 && !conflicted
5490 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5491 .await
5492 .ok()
5493 .as_deref()
5494 == Some(head.as_str())
5495 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5496 && git::is_clean(&winner.worktree).await?
5497 {
5498 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5499 if self.state.base_sync.is_none() {
5500 self.state.base_sync = Some(BaseSync {
5501 tip: saved_tip.clone(),
5502 behind: 0,
5503 attempts: 0,
5504 conflict: None,
5505 already_in: None,
5506 });
5507 }
5508 self.state.reserve_base_operation(&head, &saved_tip);
5509 self.state.event(
5510 "base-recovery",
5511 "imported standing legacy rebase as a spent synchronization reservation",
5512 );
5513 self.state.save()?;
5514 pending = self.state.base_sync_recovery.operations.last().cloned();
5515 }
5516 let expected = pending.as_ref().map(|op| op.tip.as_str());
5517 if expected.is_none()
5518 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5519 {
5520 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5521 if self.state.base_sync.is_none() {
5522 self.state.base_sync = Some(BaseSync {
5523 tip: tip.clone(),
5524 behind: 0,
5525 attempts: 0,
5526 conflict: None,
5527 already_in: None,
5528 });
5529 }
5530 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5531 self.state.event("land", why);
5532 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5533 .await?;
5534 return Ok(());
5535 }
5536 }
5537 if let Some(op) = &pending {
5538 let mut replay_proven =
5540 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5541 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5542 let wait = repository_wait(&self.state, None);
5543 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5544 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5545 Ok(true) => {
5546 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5547 self.state.save()?;
5548 git::worktree_remove(&repo, &scratch).await.ok();
5549 replay_proven = true;
5550 }
5551 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5552 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5553 }
5554 }
5555 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5556 if self.state.foreign_checkout.is_some() {
5557 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5558 .await
5559 .ok()
5560 .as_deref()
5561 == Some(op.head.as_str())
5562 && git::rev_parse(&repo, &format!("{head}^2"))
5563 .await
5564 .ok()
5565 .as_deref()
5566 == Some(op.tip.as_str());
5567 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5568 .await
5569 .is_ok_and(|m| m.is_empty())
5570 && let Ok(added) =
5571 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5572 {
5573 replay_proven = true;
5574 for commit in added {
5575 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5576 replay_proven = false;
5577 break;
5578 }
5579 }
5580 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5581 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5582 }
5583 }
5584 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5588 || (head != op.head
5589 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5590 .await?
5591 .is_empty())
5592 || (!git::is_clean(&winner.worktree).await?
5593 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5594 .await
5595 .is_ok_and(|o| o.ok()))
5596 || !replay_proven
5597 {
5598 let why = format!(
5599 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5600 op.id
5601 );
5602 let sync = self
5603 .state
5604 .base_sync
5605 .as_mut()
5606 .context("reserved synchronization has no lifetime accounting")?;
5607 sync.conflict = Some(why.clone());
5608 self.state.event("land", why);
5609 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5610 .await?;
5611 return Ok(());
5612 }
5613 if head != op.head {
5614 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5615 }
5616 tip = op.tip.clone();
5617 }
5618 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5619 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5620 if pending.is_none()
5621 && !conflicted
5622 && fetched_base
5623 && self
5624 .state
5625 .base_sync_recovery
5626 .episodes
5627 .last()
5628 .is_some_and(|e| e.boundary.is_some())
5629 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5630 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5631 && owner.execution_runs().last() == Some(&self.state.id)
5632 && owner.status != crate::queue::TaskStatus::Done
5633 && !owner.fresh_start
5634 && owner.review_branch.is_none()
5635 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5636 && let Some(approval) = owner.resumption.as_ref()
5637 && approval.instruction == owner.instruction
5638 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5639 && git::is_clean(&winner.worktree).await?
5640 && self.state.renew_base_episode(&task, approval, &head, &tip)
5641 {
5642 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5643 self.state.save()?;
5644 }
5645
5646 if (behind > 0 || conflicted || head == tip)
5655 && self
5656 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5657 .await?
5658 {
5659 return Ok(());
5660 }
5661 if conflicted && self.state.base_recovery_pending() {
5662 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5663 let owner =
5664 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5665 let owner_valid = owner.as_ref().is_ok_and(|t| {
5666 t.execution_runs().last() == Some(&self.state.id)
5667 && t.status != crate::queue::TaskStatus::Done
5668 && !t.fresh_start
5669 && t.review_branch.is_none()
5670 });
5671 let invalidation = if !owner_valid {
5672 Some("latest task ownership changed")
5673 } else if !owner.as_ref().is_ok_and(|t| {
5674 authorization
5675 .generation
5676 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5677 }) {
5678 Some("owner approval generation changed")
5679 } else if authorization.question.as_ref().is_some_and(|question| {
5680 !crate::base_recovery_control::decision_current(
5681 &crate::queue::Queue::at(crate::run::home().join("queue")),
5682 &self.state,
5683 &authorization.task,
5684 question,
5685 )
5686 }) {
5687 Some("recovery decision ownership, routing or stop evidence changed")
5688 } else if authorization.request.is_some()
5689 && !owner
5690 .as_ref()
5691 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5692 {
5693 Some("owner decision changed")
5694 } else if let Some(scope) = &authorization.scope {
5695 if scope.tip != tip {
5696 Some("fetched base changed")
5697 } else if !self
5698 .state
5699 .base_sync_recovery
5700 .episodes
5701 .last()
5702 .is_some_and(|e| e.id == scope.episode)
5703 {
5704 Some("active synchronization interval changed")
5705 } else if !owner
5706 .as_ref()
5707 .is_ok_and(|t| t.instruction == scope.instruction)
5708 {
5709 Some("task instruction changed")
5710 } else {
5711 None
5712 }
5713 } else {
5714 None
5715 };
5716 if let Some(reason) = invalidation {
5717 let authorization = self
5718 .state
5719 .base_sync_recovery
5720 .authorizations
5721 .last_mut()
5722 .unwrap();
5723 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5724 self.state.save()?;
5725 return Ok(());
5726 }
5727 if !fetched_base
5728 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5729 || !git::is_clean(&winner.worktree).await?
5730 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5731 {
5732 return Ok(());
5733 }
5734 }
5735 let recovery = conflicted && self.state.consume_base_recovery(&head);
5736 if conflicted {
5737 self.state.save()?;
5739 if !recovery {
5740 return Ok(());
5741 }
5742 }
5743
5744 if behind == 0 {
5745 if let Some(from) = self
5754 .state
5755 .rebase_fixes
5756 .iter()
5757 .rev()
5758 .find_map(|r| r.from.clone())
5759 && from != head
5760 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5761 .await
5762 .is_ok_and(|o| o.ok())
5763 {
5764 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5765 }
5766 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5772 if let Some(why) = &conflict {
5773 self.state.status = RunStatus::Blocked;
5774 self.state.event("land", why.clone());
5775 }
5776 self.state.base_sync = Some(BaseSync {
5777 tip,
5778 behind: 0,
5779 attempts,
5780 conflict,
5781 already_in: None,
5782 });
5783 if pending.is_some() {
5784 self.state.finish_base_operation(Some(&head));
5785 }
5786 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5787 .await?;
5788 self.state.save()?;
5789 return Ok(());
5790 }
5791
5792 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5793 let why = format!(
5794 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5795 synchronization(s); synchronizing again would only race it",
5796 winner.branch
5797 );
5798 self.state.status = RunStatus::Blocked;
5799 self.state.base_sync = Some(BaseSync {
5800 tip,
5801 behind,
5802 attempts,
5803 conflict: Some(why.clone()),
5804 already_in: None,
5805 });
5806 self.state.event("land", why);
5807 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5808 .await?;
5809 self.state.save()?;
5810 return Ok(());
5811 }
5812
5813 if self.state.foreign_checkout.is_some() {
5814 let wait = repository_wait(&self.state, None);
5817 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5818 if self.state.base_sync.is_none() {
5819 self.state.base_sync = Some(BaseSync {
5820 tip: tip.clone(),
5821 behind,
5822 attempts: 0,
5823 conflict: None,
5824 already_in: None,
5825 });
5826 }
5827 self.state.reserve_base_operation(&head, &tip);
5828 self.state.save()?;
5829 let merged = git::git_raw(
5830 &winner.worktree,
5831 &[
5832 "-c",
5833 "core.hooksPath=/dev/null",
5834 "merge",
5835 "--no-ff",
5836 "--no-autostash",
5837 "--no-edit",
5838 "-m",
5839 &format!("Merge {tracking} before verifying"),
5840 &tip,
5841 ],
5842 )
5843 .await?;
5844 let conflict = if merged.ok() {
5845 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5846 self.state.event(
5847 "land",
5848 format!("merged {tracking} into {} before verifying", winner.branch),
5849 );
5850 None
5851 } else {
5852 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5853 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5854 }
5855 let why = format!(
5856 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5857 winner.branch,
5858 merged.stderr.chars().take(600).collect::<String>()
5859 );
5860 self.state.status = RunStatus::Blocked;
5861 self.state.event("land", &why);
5862 Some(why)
5863 };
5864 let synced = conflict.is_none();
5865 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5866 self.state
5867 .finish_base_operation(synced.then_some(fresh.as_str()));
5868 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5869 self.state.base_sync = Some(BaseSync {
5870 tip,
5871 behind: if synced { 0 } else { behind },
5872 attempts,
5873 conflict,
5874 already_in: None,
5875 });
5876 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5877 .await?;
5878 self.state.save()?;
5879 return Ok(());
5880 }
5881
5882 self.state.event(
5883 "land",
5884 format!(
5885 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5886 winner.branch
5887 ),
5888 );
5889 self.state.save()?;
5890
5891 let branch_tracking = format!("{remote}/{}", winner.branch);
5897 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5898 let remote_tip = if self.state.task_pr_branch().is_none()
5899 && matches!(&fetched_branch, Ok(o) if o.ok())
5900 {
5901 git::rev_parse(&repo, &branch_tracking).await.ok()
5902 } else {
5903 None
5904 };
5905 if let Some(theirs) = &remote_tip
5909 && !git::is_ancestor(&repo, theirs, &head).await
5910 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5911 .await
5912 .is_ok_and(|missing| missing.is_empty())
5913 {
5914 let why = format!(
5915 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5916 them",
5917 short(theirs),
5918 winner.branch
5919 );
5920 self.state.status = RunStatus::Blocked;
5921 self.state.base_sync = Some(BaseSync {
5922 tip,
5923 behind,
5924 attempts,
5925 conflict: Some(why.clone()),
5926 already_in: None,
5927 });
5928 self.state.event("land", why);
5929 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5930 .await?;
5931 self.state.save()?;
5932 return Ok(());
5933 }
5934
5935 if self.state.base_sync.is_none() {
5936 self.state.base_sync = Some(BaseSync {
5937 tip: tip.clone(),
5938 behind,
5939 attempts: 0,
5940 conflict: None,
5941 already_in: None,
5942 });
5943 }
5944 self.state.reserve_base_operation(&head, &tip);
5945 self.state.save()?;
5946 let scratch = self.state.dir().join("base-sync");
5947 let rebased =
5948 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5949 .await
5950 {
5951 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5952 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5953 Err(e) => Err(e),
5954 };
5955 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5956 let stop_kind = match &rebased {
5957 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5958 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5959 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5960 };
5961 match rebased {
5962 Ok(None) => {
5963 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5967 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5968 self.state.finish_base_operation(Some(&fresh));
5969 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5970 let mut conflict = None;
5971 if let Some(pinned) = &remote_tip {
5972 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5973 match pushed {
5974 Ok(o) if o.ok() => self.state.event(
5975 "land",
5976 format!("pushed rebased {} to {remote}", winner.branch),
5977 ),
5978 Ok(o) => {
5979 conflict = Some(format!(
5980 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5981 winner.branch,
5982 short(pinned),
5983 o.stderr.chars().take(600).collect::<String>()
5984 ));
5985 }
5986 Err(e) => {
5987 conflict = Some(format!(
5988 "rebased {} locally but could not push it: {e:#}",
5989 winner.branch
5990 ));
5991 }
5992 }
5993 }
5994 if let Some(why) = &conflict {
5995 self.state.status = RunStatus::Blocked;
5996 self.state.event("land", why.clone());
5997 }
5998 self.state.base_sync = Some(BaseSync {
5999 tip: tip.clone(),
6000 behind: 0,
6001 attempts,
6002 conflict,
6003 already_in: None,
6004 });
6005 self.state
6006 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6007 }
6008 Ok(Some(conflict)) => {
6009 self.state.finish_base_operation(None);
6010 let why = format!(
6011 "{} conflicts with {tracking} and did not rebase: {}",
6012 winner.branch,
6013 conflict.chars().take(600).collect::<String>()
6014 );
6015 self.state.status = RunStatus::Blocked;
6016 self.state.base_sync = Some(BaseSync {
6017 tip,
6018 behind,
6019 attempts,
6020 conflict: Some(why.clone()),
6021 already_in: None,
6022 });
6023 self.state.event("land", why);
6024 }
6025 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6029 self.state.save()?;
6030 return Ok(());
6031 }
6032 Err(e) => {
6033 self.state.finish_base_operation(None);
6034 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6035 self.state.status = RunStatus::Blocked;
6036 self.state.base_sync = Some(BaseSync {
6037 tip,
6038 behind,
6039 attempts,
6040 conflict: Some(why.clone()),
6041 already_in: None,
6042 });
6043 self.state.event("land", why);
6044 }
6045 }
6046 if !self.state.base_sync_blocked() {
6047 self.state.base_sync_recovery.stop = None;
6048 }
6049 self.hold_base_sync_kind(stop_kind).await?;
6050 self.state.save()?;
6051 Ok(())
6052 }
6053
6054 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6060 if self.state.task_pr_branch().is_some() {
6063 return None;
6064 }
6065 let repo = self.state.repo.clone();
6066 let remote = self.state.config.merge.remote.clone();
6067 let fetched = git::fetch(&repo, &remote, branch).await;
6068 if !matches!(&fetched, Ok(o) if o.ok()) {
6069 return None;
6070 }
6071 let branch_tracking = format!("{remote}/{branch}");
6072 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6073 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6074 return None;
6075 }
6076 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6077 .await
6078 .is_ok_and(|missing| missing.is_empty())
6079 {
6080 return Some(format!(
6081 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6082 them",
6083 short(&theirs)
6084 ));
6085 }
6086 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6087 Ok(o) if o.ok() => {
6088 self.state
6089 .event("land", format!("pushed rebased {branch} to {remote}"));
6090 None
6091 }
6092 Ok(o) => Some(format!(
6093 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6094 someone may have pushed): {}",
6095 short(&theirs),
6096 o.stderr.chars().take(600).collect::<String>()
6097 )),
6098 Err(e) => Some(format!(
6099 "{branch} is rebased locally but could not be pushed: {e:#}"
6100 )),
6101 }
6102 }
6103
6104 async fn settle_already_in(
6121 &mut self,
6122 branch: &str,
6123 tip: &str,
6124 head: &str,
6125 attempts: usize,
6126 behind: usize,
6127 ) -> Result<bool> {
6128 let repo = self.state.repo.clone();
6129 let remote = self.state.config.merge.remote.clone();
6130 let start = self.state.base_commit.clone();
6131 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6132 Ok(Some(e)) => e,
6133 Ok(None) => return Ok(false),
6134 Err(e) => {
6135 tracing::warn!("already-in-base check for {branch}: {e:#}");
6136 return Ok(false);
6137 }
6138 };
6139 let mut verified = vec![head.to_owned()];
6140 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6141 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6142 if matches!(&fetched, Ok(o) if o.ok())
6143 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6144 && theirs != head
6145 {
6146 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6147 Ok(Some(_)) => verified.push(theirs),
6148 _ => return Ok(false),
6149 }
6150 }
6151 let base_branch = self.state.base_branch.clone();
6152 let message = format!(
6153 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6154 land",
6155 evidence.names(),
6156 evidence.proof.as_str()
6157 );
6158 let closed =
6159 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6160 .await;
6161 match closed {
6162 Ok(Ok(url)) => self
6163 .state
6164 .event("land", format!("closed {url}: superseded on {base_branch}")),
6165 Ok(Err(why)) => self
6166 .state
6167 .event("land", format!("did not close a pull request: {why}")),
6168 Err(e) => {
6169 let why = format!(
6170 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6171 not be closed ({e:#}); resume to retry"
6172 );
6173 self.state.status = RunStatus::Blocked;
6174 self.state.base_sync = Some(BaseSync {
6175 tip: tip.to_owned(),
6176 behind,
6177 attempts,
6178 conflict: Some(why.clone()),
6179 already_in: None,
6180 });
6181 self.state.event("land", why);
6182 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6183 .await?;
6184 self.state.save()?;
6185 return Ok(true);
6186 }
6187 }
6188 self.state.status = RunStatus::AlreadyInBase;
6189 self.state.base_sync_recovery.stop = None;
6190 self.state.base_sync = Some(BaseSync {
6191 tip: tip.to_owned(),
6192 behind,
6193 attempts,
6194 conflict: None,
6195 already_in: Some(evidence),
6196 });
6197 self.state.event("land", message);
6198 self.state.save()?;
6199 self.settle_questions();
6200 Ok(true)
6201 }
6202
6203 fn landing_base(&self) -> String {
6213 self.state
6214 .base_sync
6215 .as_ref()
6216 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6217 }
6218
6219 pub async fn fix_selected(
6252 &mut self,
6253 ids: &[String],
6254 reason: &str,
6255 allow_stale: bool,
6256 ) -> Result<()> {
6257 let reason = reason.trim();
6258 if reason.is_empty() {
6259 bail!("a fix request needs a reason — that is the operator's own record of why");
6260 }
6261 if ids.is_empty() {
6262 bail!("no finding id given");
6263 }
6264 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6265 bail!(
6266 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6267 has already concluded — can be given a targeted fix. A run still \
6268 in progress should simply be resumed; a `merged` run's branch has \
6269 already landed, so its answer is a fresh `magi review <branch>`, \
6270 not reopening this run's own record",
6271 self.state.id,
6272 self.state.status.as_str()
6273 );
6274 }
6275 let _claim = FixClaim::acquire(&self.state.dir())?;
6278 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6279 let Some(winner) = self.state.winner().cloned() else {
6280 bail!("run {} has no winning candidate to fix", self.state.id);
6281 };
6282 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6283 bail!(
6284 "branch `{}` no longer exists; this run cannot be extended",
6285 winner.branch
6286 );
6287 }
6288 let home = crate::run::home();
6289 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6290 bail!(
6291 "run {} is currently being worked on by another magi process",
6292 self.state.id
6293 );
6294 }
6295 let mut seen = BTreeSet::new();
6299 let mut findings = Vec::new();
6300 let mut missing = Vec::new();
6301 for id in ids {
6302 if !seen.insert(id.clone()) {
6303 continue;
6304 }
6305 match self.state.finding(id) {
6306 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6307 id: f.id.clone(),
6308 severity: f.severity,
6309 reviewer_vote: rec.vote,
6310 round: round.round,
6311 round_head: round.head.clone(),
6312 reviewer: rec.reviewer,
6313 agent: rec.agent.clone(),
6314 file: f.file.clone(),
6315 line: f.line,
6316 title: f.title.clone(),
6317 detail: f.detail.clone(),
6318 outcome: OperatorFixOutcome::Pending,
6319 }),
6320 None => missing.push(id.clone()),
6321 }
6322 }
6323 if !missing.is_empty() {
6324 bail!(
6325 "unknown finding id(s): {}; nothing was changed",
6326 missing.join(", ")
6327 );
6328 }
6329
6330 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6331 let stale_details: Vec<(String, String)> = findings
6332 .iter()
6333 .filter(|f| f.round_head != head_at_request)
6334 .map(|f| (f.id.clone(), f.round_head.clone()))
6335 .collect();
6336 let stale = !stale_details.is_empty();
6337 if stale && !allow_stale {
6338 bail!(
6339 "the branch has moved since some finding(s) were raised — {} — now \
6340 at {}; pass --allow-stale to fix anyway, or re-run review first",
6341 stale_details
6342 .iter()
6343 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6344 .collect::<Vec<_>>()
6345 .join(", "),
6346 short(&head_at_request)
6347 );
6348 }
6349
6350 let request = OperatorFixRequest {
6351 requested_at: Timestamp::now(),
6352 reason: reason.to_owned(),
6353 findings,
6354 head_at_request: head_at_request.clone(),
6355 allow_stale,
6356 stale,
6357 fix: None,
6358 result_head: None,
6359 follow_up_review_run: None,
6360 };
6361 self.state.event(
6362 "fix",
6363 format!(
6364 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6365 request.findings.len(),
6366 request
6367 .findings
6368 .iter()
6369 .map(|f| f.id.as_str())
6370 .collect::<Vec<_>>()
6371 .join(", "),
6372 ),
6373 );
6374 self.state.operator_fixes.push(request);
6381 self.state.save()?;
6382 let request_index = self.state.operator_fixes.len() - 1;
6383
6384 let repo = self.state.repo.clone();
6393 let coordination = self.coordinate(&repo).await?;
6394 if winner.worktree.exists() {
6395 let dirty = git::git(
6398 &winner.worktree,
6399 &["status", "--porcelain", "--untracked-files=all"],
6400 )
6401 .await?;
6402 let only_withheld = dirty.lines().all(|l| {
6403 l.strip_prefix("?? ")
6404 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6405 });
6406 if !only_withheld {
6407 bail!(
6408 "`{}` has uncommitted changes; refusing to touch it — commit or \
6409 discard them first",
6410 winner.worktree.display()
6411 );
6412 }
6413 git::worktree_remove(&self.state.repo, &winner.worktree)
6414 .await
6415 .ok();
6416 }
6417 let fix_worktree = self.state.worktree_root().join("operator-fix");
6418 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6419 git::git(
6420 &self.state.repo,
6421 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6422 )
6423 .await
6424 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6425 if !git::is_clean(&fix_worktree).await? {
6426 git::worktree_remove(&self.state.repo, &fix_worktree)
6427 .await
6428 .ok();
6429 bail!(
6430 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6431 winner.branch
6432 );
6433 }
6434 let repo_for_setup = self.state.repo.clone();
6435 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6436 Ok(()) => {
6437 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6438 }
6439 Err(e) => Err(e),
6440 };
6441 if let Err(e) = setup {
6442 git::worktree_remove(&self.state.repo, &fix_worktree)
6443 .await
6444 .ok();
6445 return Err(e);
6446 }
6447
6448 self.state.resource_block = None;
6451
6452 let run_id = self.state.id.clone();
6453 let prompts = self.state.config.prompts.clone();
6454 let language = self.state.config.graph.language.clone();
6455 let sessions = self.state.config.graph.sessions;
6456 let artifacts = agent::artifacts_dir(&self.state.dir());
6457 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6458 .findings
6459 .iter()
6460 .map(|f| Finding {
6461 id: f.id.clone(),
6462 severity: f.severity,
6463 file: f.file.clone(),
6464 line: f.line,
6465 title: f.title.clone(),
6466 detail: f.detail.clone(),
6467 })
6468 .collect();
6469 let fix_prompt = prompt::operator_fix_in(
6470 &self.state.instruction,
6471 &finding_list,
6472 reason,
6473 &stale_details,
6474 &head_at_request,
6475 &language,
6476 &self.state.config.graph.github_language,
6477 );
6478 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6479 drop(coordination);
6480 let (job, seat, out) = self
6481 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6482 prompt: fix_prompt.clone(),
6483 spec,
6484 seat,
6485 cwd: fix_worktree.clone(),
6486 timeout,
6487 allow_write: true,
6488 sessions,
6489 artifacts: artifacts.clone(),
6490 stem: "operator-fix".to_owned(),
6491 handover: None,
6492 })
6493 .await;
6494 let agent_id = seat.agent.clone();
6495
6496 let mut fix = FixRecord {
6497 agent: agent_id,
6498 addressed: Vec::new(),
6499 rejected: Vec::new(),
6500 notes: String::new(),
6501 committed: false,
6502 failed: None,
6503 duration_ms: 0,
6504 continuation: None,
6505 };
6506 let mut final_seat = seat.clone();
6507 match out {
6508 AgentOutcome::Ok(o) => {
6509 fix.duration_ms = o.duration_ms;
6510 let parsed = verdict::extract_json::<FixReport>(&o.text);
6511 let incomplete_reason = match &parsed {
6512 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6513 "the reply parsed, but it reported a command whose own CLI \
6514 never confirmed an exit status"
6515 .to_owned(),
6516 ),
6517 Ok(_) => None,
6518 Err(e) => Some(e.to_string()),
6519 };
6520 match incomplete_reason {
6521 None => {
6522 let report = parsed.expect("checked Ok above");
6523 fix.addressed = report.addressed;
6524 fix.rejected = report.rejected;
6525 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6526 }
6527 Some(reason) => {
6528 let (resumed_seat, resolved, failure, cont) = self
6529 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6530 .await;
6531 fix.duration_ms += cont.cumulative_wait_ms;
6532 fix.continuation = Some(cont);
6533 final_seat = resumed_seat;
6534 match resolved {
6535 Some(report) => {
6536 fix.addressed = report.addressed;
6537 fix.rejected = report.rejected;
6538 fix.notes =
6539 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6540 }
6541 None => fix.failed = failure,
6542 }
6543 }
6544 }
6545 }
6546 AgentOutcome::Dropped(o) => {
6547 fix.duration_ms = o.duration_ms;
6548 let why = o
6549 .dropped
6550 .as_ref()
6551 .map(|d| d.why.as_str())
6552 .unwrap_or("the CLI ended the stream without delivering its answer");
6553 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6554 }
6555 AgentOutcome::Quota(o) => {
6556 self.state.quota.push(QuotaLoss::from_report(
6557 final_seat.key.clone(),
6558 "fix",
6559 o.quota.as_ref(),
6560 ));
6561 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6562 }
6563 AgentOutcome::Failed(e) => fix.failed = Some(e),
6564 AgentOutcome::ResourceBlocked(e) => {
6565 fix.failed = Some(format!("the fixer was not started: {e}"));
6566 }
6567 }
6568 if fix.continuation.is_none() {
6571 fix.continuation = Some(ContinuationRecord::not_needed());
6572 }
6573 self.state.seats.insert(final_seat.key.clone(), final_seat);
6574
6575 let rescue_message = format!(
6576 "magi: operator-selected fix ({}) (uncommitted work)",
6577 self.state.operator_fixes[request_index]
6578 .findings
6579 .iter()
6580 .map(|f| f.id.as_str())
6581 .collect::<Vec<_>>()
6582 .join(", ")
6583 );
6584 let repo = self.state.repo.clone();
6585 let coordination = self.coordinate(&repo).await?;
6586 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6587 self.state.note_withheld("fix", &r.withheld);
6588 }
6589 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6590 fix.committed = after != head_at_request;
6591 git::worktree_remove(&self.state.repo, &fix_worktree)
6592 .await
6593 .ok();
6594 drop(coordination);
6595
6596 self.state.event(
6597 "fix",
6598 match &fix.failed {
6599 Some(reason) => format!(
6600 "operator fix: adoption report was lost ({reason}); {}",
6601 if fix.committed {
6602 "committed"
6603 } else {
6604 "NO new commit"
6605 }
6606 ),
6607 None => format!(
6608 "operator fix: {} addressed, {} rejected, {}",
6609 fix.addressed.len(),
6610 fix.rejected.len(),
6611 if fix.committed {
6612 "committed"
6613 } else {
6614 "NO new commit"
6615 }
6616 ),
6617 },
6618 );
6619
6620 for f in &mut self.state.operator_fixes[request_index].findings {
6627 f.outcome = if fix.failed.is_some() {
6628 OperatorFixOutcome::Unreported
6629 } else if fix.addressed.contains(&f.id) {
6630 OperatorFixOutcome::Addressed
6631 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6632 OperatorFixOutcome::Rejected { why: r.why.clone() }
6633 } else {
6634 OperatorFixOutcome::Unreported
6635 };
6636 }
6637
6638 let committed = fix.committed;
6639 if committed {
6640 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6641 }
6642 self.state.operator_fixes[request_index].fix = Some(fix);
6643 self.state.save()?;
6646 if self.state.resource_block.is_some() {
6647 return self.stop_for_resource();
6648 }
6649
6650 if committed {
6651 self.state.event(
6652 "fix",
6653 format!(
6654 "operator fix committed {}; opening a follow-up review-only run",
6655 short(&after)
6656 ),
6657 );
6658 let origin =
6661 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6662 match Self::review(
6663 &self.state.repo,
6664 &winner.branch,
6665 self.state.config.clone(),
6666 origin,
6667 )
6668 .await
6669 {
6670 Ok(mut follow_up) => {
6671 follow_up.state.event(
6672 "start",
6673 format!(
6674 "requested by an operator fix on run {} for finding(s) {}",
6675 self.state.id,
6676 self.state.operator_fixes[request_index]
6677 .findings
6678 .iter()
6679 .map(|f| f.id.as_str())
6680 .collect::<Vec<_>>()
6681 .join(", "),
6682 ),
6683 );
6684 follow_up.state.save()?;
6685 let follow_up_id = follow_up.state.id.clone();
6686 let adopted = match crate::direct::adopt(
6689 &follow_up.state,
6690 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6691 ) {
6692 Ok(a) => a,
6693 Err(e) => {
6694 self.state.event(
6697 "fix",
6698 format!(
6699 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6700 ),
6701 );
6702 self.state.save()?;
6703 return Ok(());
6704 }
6705 };
6706 let executed = follow_up.execute().await;
6707 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6708 if let Some(a) = adopted {
6709 a.finish(&follow_up.state, executed);
6710 }
6711 if let Some(e) = failure {
6712 self.state.event(
6713 "fix",
6714 format!(
6715 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6716 ),
6717 );
6718 }
6719 self.state.operator_fixes[request_index].follow_up_review_run =
6720 Some(follow_up_id);
6721 }
6722 Err(e) => {
6723 self.state.event(
6724 "fix",
6725 format!("committed the fix but could not open a follow-up review: {e:#}"),
6726 );
6727 }
6728 }
6729 self.state.save()?;
6730 }
6731
6732 Ok(())
6733 }
6734
6735 async fn ask_fixer(
6755 &mut self,
6756 winner: &Candidate,
6757 node: &'static str,
6758 round: Option<usize>,
6759 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6760 ) -> (SeatJob, SeatState, AgentOutcome) {
6761 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6762 self.ask_fixer_among(attempts, node, round, None, build)
6763 .await
6764 }
6765
6766 async fn ask_fixer_among(
6771 &mut self,
6772 attempts: Vec<(AgentSpec, String)>,
6773 node: &'static str,
6774 round: Option<usize>,
6775 restore: Option<(&Path, &str)>,
6776 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6777 ) -> (SeatJob, SeatState, AgentOutcome) {
6778 let run_id = self.state.id.clone();
6779 let prompts = self.state.config.prompts.clone();
6780 let cache = self.state.config.cache_dir();
6781 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6782 let mut last = None;
6783 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6784 let seat = self.seat(&key, &spec.id);
6785 let mut job = build(spec.clone(), seat);
6786 if i > 0 {
6787 job.stem = format!("{}-{}", job.stem, spec.id);
6788 }
6789 let ctx = WaveCtx {
6790 carry_seats: false,
6791 pauses: Some((&self.pause, &self.interrupt)),
6792 run: &run_id,
6793 node,
6794 prompts: &prompts,
6795 cache: cache.as_deref(),
6796 round,
6797 };
6798 let (seat, out) =
6799 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6800 let advances = match &out {
6803 AgentOutcome::Ok(o) => agent::output_advances(o),
6804 AgentOutcome::ResourceBlocked(_) => false,
6805 _ => true,
6806 };
6807 if let Some(next) = ids.get(i + 1)
6808 && advances
6809 {
6810 self.state.seats.insert(seat.key.clone(), seat.clone());
6811 let class =
6812 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6813 let reason = match &out {
6814 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6815 other => fail_reason(other),
6816 };
6817 if let Some((wt, rev)) = restore
6821 && let Err(e) = self.graft_restore(wt, rev).await
6822 {
6823 let why =
6824 format!("could not restore the worktree before the next fixer: {e:#}");
6825 tracing::warn!("{why}");
6826 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6827 AgentOutcome::ResourceBlocked(why)
6828 } else {
6829 AgentOutcome::Failed(why)
6830 };
6831 last = Some((job, seat, out));
6832 break;
6833 }
6834 record_handover(
6835 &mut self.state,
6836 node,
6837 &seat.key,
6838 &spec.id,
6839 next,
6840 &class,
6841 &reason,
6842 );
6843 continue;
6844 }
6845 last = Some((job, seat, out));
6846 break;
6847 }
6848 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6849 if let AgentOutcome::Quota(output) = &out
6850 && node != "graft"
6851 {
6852 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6857 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6858 self.state.quota.push(loss);
6859 self.state.seats.insert(seat.key.clone(), seat.clone());
6860 self.state.status = RunStatus::Blocked;
6861 return (
6862 job,
6863 seat,
6864 AgentOutcome::ResourceBlocked(
6865 "fixer unavailable; restore agent availability and resume".into(),
6866 ),
6867 );
6868 }
6869 (job, seat, out)
6870 }
6871
6872 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6883 use crate::recovery::{Mode, Repair};
6884 if !self.state.reviews.is_empty() {
6885 return Ok(());
6886 }
6887 let Some(mut rec) = self.state.recovery.clone() else {
6888 return Ok(());
6889 };
6890 if rec.mode != Mode::VerifyRepair {
6891 return Ok(());
6892 }
6893 let wt = winner.worktree.clone();
6894 let mut head = git::rev_parse(&wt, "HEAD").await?;
6895 match rec.repair.clone() {
6896 Repair::Done { .. } => return Ok(()),
6897 Repair::WaitingForRestore { before } => {
6898 self.graft_restore(&wt, &before).await?;
6899 head = before;
6900 rec.repair = Repair::NotStarted;
6901 self.state.recovery = Some(rec.clone());
6902 self.state.save()?;
6903 }
6904 Repair::Started { before } => {
6905 if let Err(e) = self.graft_restore(&wt, &before).await {
6909 if e.is::<crate::repo_lock::ResourceBlocked>() {
6910 return Err(e);
6911 }
6912 return self
6913 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6914 .await;
6915 }
6916 rec.repair = Repair::Done {
6917 before: before.clone(),
6918 after: before,
6919 committed: false,
6920 failed: Some("interrupted; rolled back".to_owned()),
6921 };
6922 self.state
6923 .event("recovery", "the repair was interrupted and rolled back");
6924 self.state.recovery = Some(rec);
6925 self.state.save()?;
6926 return Ok(());
6927 }
6928 Repair::NotStarted => {}
6929 }
6930 if crate::recovery::recheck(&mut rec, &head) {
6931 self.state.event("recovery", rec.summary());
6932 self.state.recovery = Some(rec);
6933 self.state.save()?;
6934 return Ok(());
6935 }
6936 rec.repair = Repair::Started {
6937 before: head.clone(),
6938 };
6939 self.state.recovery = Some(rec.clone());
6940 self.state.save()?;
6941
6942 let language = self.state.config.graph.language.clone();
6943 let sessions = self.state.config.graph.sessions;
6944 let artifacts = agent::artifacts_dir(&self.state.dir());
6945 let fix_prompt = prompt::recovery_fix(
6946 &self.state.instruction,
6947 &rec,
6948 self.state.config.graph.review_rounds,
6949 &language,
6950 );
6951 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6952 self.state.event(
6953 "recovery",
6954 format!(
6955 "repair: the fixer starts from the failed verification on {}",
6956 short(&head)
6957 ),
6958 );
6959 let (_job, seat, out) = self
6960 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6961 prompt: fix_prompt.clone(),
6962 spec,
6963 seat,
6964 cwd: wt.clone(),
6965 timeout,
6966 allow_write: true,
6967 sessions,
6968 artifacts: artifacts.clone(),
6969 stem: "fix-recovery".to_owned(),
6970 handover: None,
6971 })
6972 .await;
6973 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6974 rec.repair = Repair::WaitingForRestore { before: head };
6975 self.state.recovery = Some(rec);
6976 return self.stop_for_resource();
6977 }
6978 let failed = match &out {
6979 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6980 AgentOutcome::Quota(o) => {
6981 self.state.quota.push(QuotaLoss::from_report(
6982 seat.key.clone(),
6983 "fix",
6984 o.quota.as_ref(),
6985 ));
6986 Some("rate limited (quota); the fixer could not run".to_owned())
6987 }
6988 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6989 AgentOutcome::Failed(e) => Some(e.clone()),
6990 };
6991 self.state.seats.insert(seat.key.clone(), seat);
6992 let coordination = self.coordinate(&wt).await?;
6993 let failed = match failed {
6994 Some(why) => {
6996 if let Err(e) = git::restore_to(&wt, &head).await {
6997 return self
6998 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6999 .await;
7000 }
7001 Some(why)
7002 }
7003 None => {
7004 if let Ok(r) = git::rescue_commit(
7005 &wt,
7006 "magi: recovery repair of the failed verification (uncommitted work)",
7007 )
7008 .await
7009 {
7010 self.state.note_withheld("fix", &r.withheld);
7011 }
7012 None
7013 }
7014 };
7015 let after = git::rev_parse(&wt, "HEAD").await?;
7016 drop(coordination);
7017 self.state.event(
7018 "recovery",
7019 match &failed {
7020 Some(why) => format!("repair: nothing kept ({why})"),
7021 None if after != head => {
7022 format!("repair: committed {}", short(&after))
7023 }
7024 None => "repair: the fixer made no commit".to_owned(),
7025 },
7026 );
7027 rec.repair = Repair::Done {
7028 before: head.clone(),
7029 committed: after != head,
7030 after,
7031 failed,
7032 };
7033 self.state.recovery = Some(rec);
7034 self.state.save()?;
7035 Ok(())
7036 }
7037
7038 async fn stop_recovery(
7041 &mut self,
7042 mut rec: crate::recovery::RecoveryRecord,
7043 head: &str,
7044 why: String,
7045 ) -> Result<()> {
7046 rec.repair = crate::recovery::Repair::Started {
7048 before: head.to_owned(),
7049 };
7050 self.state.recovery = Some(rec);
7051 self.state
7052 .event("recovery", format!("repair could not be undone: {why}"));
7053 self.state.status = RunStatus::Blocked;
7054 self.state.save()?;
7055 bail!("recovery repair could not be rolled back: {why}")
7056 }
7057
7058 async fn review_loop(&mut self) -> Result<()> {
7059 if self.hold_base_sync().await? {
7060 return Ok(());
7061 }
7062 loop {
7063 if self
7064 .state
7065 .base_sync
7066 .as_ref()
7067 .is_some_and(|s| s.conflict.is_some())
7068 {
7069 return Ok(());
7070 }
7071 if self.catchup_repairable() {
7072 let record = self.state.reviews.last().expect("repairable round").clone();
7073 let winner = self
7074 .state
7075 .winner()
7076 .cloned()
7077 .context("catch-up repair has no winner")?;
7078 if !self.fix_review_round(&winner, record).await? {
7079 return Ok(());
7080 }
7081 }
7082 self.review_loop_pass().await?;
7083 if !self.catchup_repairable() {
7084 return Ok(());
7085 }
7086 }
7087 }
7088
7089 fn catchup_repairable(&self) -> bool {
7092 self.state.reviews.len() < self.state.config.graph.review_rounds
7093 && self.state.reviews.last().is_some_and(|r| {
7094 !r.clean
7095 && r.blocking == 0
7096 && r.fix.is_none()
7097 && r.e2e_status() == E2eStatus::Failed
7098 })
7099 }
7100
7101 async fn review_loop_pass(&mut self) -> Result<()> {
7102 if self
7107 .state
7108 .base_sync
7109 .as_ref()
7110 .is_some_and(|s| s.conflict.is_some())
7111 {
7112 return Ok(());
7113 }
7114 let run_id = self.state.id.clone();
7119 let prompts = self.state.config.prompts.clone();
7120 let Some(winner) = self.state.winner().cloned() else {
7121 return Ok(());
7122 };
7123 let granted_review = self
7124 .state
7125 .repair_grants
7126 .last()
7127 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7128 .cloned();
7129 let fresh_review = granted_review
7130 .as_ref()
7131 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7132 let max_rounds = granted_review
7133 .as_ref()
7134 .map_or(self.state.config.graph.review_rounds, |g| {
7135 g.review_start.saturating_add(1)
7136 });
7137 if let Some(status) =
7147 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7148 {
7149 if status == RunStatus::Gating {
7152 self.record_contested_handoff();
7153 }
7154 self.state.status = status;
7155 self.state.save()?;
7156 return Ok(());
7157 }
7158 self.state.status = RunStatus::Reviewing;
7159 if !fresh_review
7169 && self.state.reviews.last().is_some_and(|r| {
7170 r.e2e_status() == E2eStatus::ResourceBlocked
7171 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7172 })
7173 {
7174 let shell = self.state.config.shell();
7175 return self
7176 .stop_unresolved_review(
7177 "the last round's own verification never resolved",
7178 &shell,
7179 &winner.worktree,
7180 )
7181 .await;
7182 }
7183
7184 if granted_review.is_none() {
7185 self.recovery_repair(&winner).await?;
7186 }
7187 if self.resource_halt()? {
7188 return Ok(());
7189 }
7190
7191 let repo = self.state.repo.clone();
7192 let root = self.state.worktree_root();
7193 let language = self.state.config.graph.language.clone();
7194 let sessions = self.state.config.graph.sessions;
7195 let artifacts = self.merge_repair_artifacts();
7196 let base = self.landing_base();
7197 let base_short = short(&base);
7198 let reviewers = self.roles.reviewers.clone();
7199 let shell = self.state.config.shell();
7200
7201 for round in (self.state.reviews.len() + 1)..=max_rounds {
7202 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7203 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7204 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7205 let prev_verification = self
7214 .state
7215 .reviews
7216 .last()
7217 .and_then(|r| r.verification_summary(&head));
7218 let prior_section = match (&self.state.recovery, round) {
7221 (Some(rec), 1) if self.state.reviews.is_empty() => {
7222 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7223 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7224 Some(crate::recovery::head_of(
7225 &git::diff(&winner.worktree, reviewed, "HEAD")
7226 .await
7227 .unwrap_or_default(),
7228 crate::recovery::SINCE_DIFF_MAX,
7229 ))
7230 }
7231 _ => None,
7232 };
7233 Some(prompt::recovery_review(rec, since.as_deref()))
7234 }
7235 _ => None,
7236 };
7237
7238 let mut jobs = Vec::new();
7242 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7243 self.coordinate(&repo).await?
7244 } else {
7245 None
7246 };
7247 for (r, spec) in reviewers.iter().cloned().enumerate() {
7248 let wt = root.join(format!("review-{}", r + 1));
7249 if wt.exists() {
7252 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7253 } else {
7254 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7255 None
7256 } else {
7257 self.coordinate(&repo).await?
7258 };
7259 git::worktree_add_detached(&repo, &wt, &head).await?;
7260 }
7261 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7262 None
7263 } else {
7264 self.coordinate(&repo).await?
7265 };
7266 self.setup_seat_worktree(&repo, &wt, false).await?;
7267 let seat_key = format!("review-{}", r + 1);
7268 let spec = pick_start_spec(
7271 &self.roles.reviewer_roster,
7272 spec,
7273 self.state.seat_history.get(&seat_key),
7274 );
7275 let seat = self.seat(&seat_key, &spec.id);
7276 jobs.push(SeatJob {
7277 prompt: prompt::review(&prompt::ReviewCtx {
7278 instruction: &self.state.instruction,
7279 branch: &winner.branch,
7280 base_short: &base_short,
7281 stat: &stat,
7282 patch: &patch,
7283 verification: prev_verification.as_ref(),
7284 prior: prior_section.as_deref(),
7285 reviewers: reviewers.len(),
7286 round,
7287 rounds: max_rounds,
7288 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7291 lens: Lens::for_seat(r),
7292 language: &language,
7293 github_language: &self.state.config.graph.github_language,
7294 }),
7295 spec,
7296 seat,
7297 cwd: wt,
7298 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7299 allow_write: false,
7300 sessions,
7301 artifacts: artifacts.clone(),
7302 stem: format!("review-{round}-{}", r + 1),
7303 handover: None,
7304 });
7305 }
7306
7307 drop(coordination);
7308 self.state.event(
7309 "review",
7310 format!(
7311 "round {round}: {} reviewers on {}",
7312 jobs.len(),
7313 short(&head)
7314 ),
7315 );
7316 let mut quota_losses = Vec::new();
7317 let review_retries = self.state.config.graph.retries;
7318 let review_cache = self.state.config.cache_dir();
7319 let ctx = WaveCtx {
7320 carry_seats: true,
7321 pauses: Some((&self.pause, &self.interrupt)),
7322 run: &run_id,
7323 node: "review",
7324 prompts: &prompts,
7325 cache: review_cache.as_deref(),
7326 round: Some(round),
7327 };
7328 let results = ask_json_wave::<Review>(
7329 jobs,
7330 Arc::clone(&self.sem),
7331 review_retries,
7332 &self.roles.reviewer_roster,
7333 &ctx,
7334 &mut quota_losses,
7335 &mut self.state,
7336 &|_: &Review| Ok(()),
7337 )
7338 .await;
7339 if self.state.resource_block.is_some() {
7340 return self.stop_for_resource();
7341 }
7342 let round_quota_missing = quota_losses.len();
7346 self.state.quota.extend(quota_losses);
7347
7348 let mut records = Vec::new();
7349 let mut all_findings = Vec::new();
7350 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7351 let agent_id = seat.agent.clone();
7352 self.state.seats.insert(seat.key.clone(), seat);
7353 let mut record = ReviewRecord {
7354 reviewer: r + 1,
7355 agent: agent_id,
7356 summary: String::new(),
7357 findings: Vec::new(),
7358 vote: None,
7359 failed: None,
7360 duration_ms: 0,
7361 attempts,
7367 };
7368 match res {
7369 Ok((review, out)) => {
7370 record.summary =
7378 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7379 record.vote = Some(review.vote);
7380 record.duration_ms = out.duration_ms;
7381 for (n, mut f) in review.findings.into_iter().enumerate() {
7382 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7385 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7386 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7387 f.file = f
7393 .file
7394 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7395 all_findings.push(f.clone());
7396 record.findings.push(f);
7397 }
7398 self.state.event(
7399 "review",
7400 format!(
7401 "round {round}: reviewer {} voted {} with {} finding(s)",
7402 r + 1,
7403 review.vote.label(),
7404 record.findings.len()
7405 ),
7406 );
7407 }
7408 Err(e) => {
7409 record.failed = Some(e.to_string());
7410 self.state.event(
7411 "review",
7412 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7413 );
7414 }
7415 }
7416 records.push(record);
7417 }
7418
7419 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7426 let vote_split =
7427 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7428 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7429 if vote_split {
7430 self.state.event(
7431 "review",
7432 format!(
7433 "round {round}: votes split ({}) — one round of reconsideration",
7434 initial_votes
7435 .iter()
7436 .map(|v| v.label())
7437 .collect::<Vec<_>>()
7438 .join(", ")
7439 ),
7440 );
7441 let panel: Vec<ReviewSeatReport<'_>> = records
7444 .iter()
7445 .filter_map(|r| {
7446 r.vote.map(|vote| ReviewSeatReport {
7447 reviewer: r.reviewer,
7448 vote,
7449 summary: &r.summary,
7450 findings: &r.findings,
7451 })
7452 })
7453 .collect();
7454
7455 let mut jobs = Vec::new();
7456 let mut seats_at = Vec::new();
7457 for (r, spec) in reviewers.iter().cloned().enumerate() {
7458 if records[r].vote.is_none() {
7462 continue;
7463 }
7464 let wt = root.join(format!("review-{}", r + 1));
7465 let seat_key = format!("review-{}", r + 1);
7466 let spec = self.occupant(&seat_key, spec);
7467 let seat = self.seat(&seat_key, &spec.id);
7468 let build = |with_patch: bool| {
7476 prompt::review_reconsider(&ReviewReconsiderCtx {
7477 instruction: &self.state.instruction,
7478 reviewer: r + 1,
7479 lens: Lens::for_seat(r),
7480 panel: &panel,
7481 patch: with_patch.then_some(ReviewPatch {
7482 branch: &winner.branch,
7483 base_short: &base_short,
7484 stat: &stat,
7485 patch: &patch,
7486 }),
7487 round,
7488 rounds: max_rounds,
7489 language: &language,
7490 })
7491 };
7492 let full = build(true);
7493 let prompt = if has_context(&spec, &seat, sessions) {
7494 build(false)
7495 } else {
7496 full.clone()
7497 };
7498 jobs.push(SeatJob {
7499 prompt,
7500 spec,
7501 seat,
7502 cwd: wt,
7503 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7504 allow_write: false,
7505 sessions,
7506 artifacts: artifacts.clone(),
7507 stem: format!("review-{round}-reconsider-{}", r + 1),
7508 handover: Some(full),
7509 });
7510 seats_at.push(r);
7511 }
7512
7513 let mut recon_quota_losses = Vec::new();
7514 let recon_cache = self.state.config.cache_dir();
7515 let recon_ctx = WaveCtx {
7516 carry_seats: true,
7517 pauses: Some((&self.pause, &self.interrupt)),
7518 run: &run_id,
7519 node: "review",
7520 prompts: &prompts,
7521 cache: recon_cache.as_deref(),
7522 round: Some(round),
7523 };
7524 let recon_results = ask_json_wave::<ReviewRevote>(
7525 jobs,
7526 Arc::clone(&self.sem),
7527 review_retries,
7528 &self.roles.reviewer_roster,
7529 &recon_ctx,
7530 &mut recon_quota_losses,
7531 &mut self.state,
7532 &|_: &ReviewRevote| Ok(()),
7533 )
7534 .await;
7535 if self.state.resource_block.is_some() {
7536 return self.stop_for_resource();
7537 }
7538 self.state.quota.extend(recon_quota_losses);
7539
7540 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7541 let agent_id = seat.agent.clone();
7542 self.state.seats.insert(seat.key.clone(), seat);
7543 let mut rec = ReviewRevoteRecord {
7544 reviewer: r + 1,
7545 agent: agent_id,
7546 vote: None,
7547 reason: String::new(),
7548 failed: None,
7549 };
7550 match res {
7551 Ok((rv, _)) => {
7552 rec.vote = Some(rv.vote);
7553 rec.reason =
7554 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7555 self.state.event(
7556 "review",
7557 format!(
7558 "round {round}: reviewer {} revoted {}",
7559 r + 1,
7560 rv.vote.label()
7561 ),
7562 );
7563 }
7564 Err(e) => {
7565 rec.failed = Some(e.to_string());
7566 self.state.event(
7567 "review",
7568 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7569 );
7570 }
7571 }
7572 reconsideration.push(rec);
7573 }
7574 } else if initial_votes.len() > 1 {
7575 self.state.event(
7576 "review",
7577 format!(
7578 "round {round}: votes agreed ({}) — no reconsideration",
7579 initial_votes[0].label()
7580 ),
7581 );
7582 }
7583
7584 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7585 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7586 let defer_e2e =
7597 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7598 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7599 let reason =
7600 format!("{blocking} blocking finding(s) already required a fix this round");
7601 self.state.event(
7602 "verify",
7603 format!(
7604 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7605 {}); it will run once a round has none left",
7606 short(&head)
7607 ),
7608 );
7609 (Vec::new(), false, true, Some(reason))
7610 } else {
7611 let e2e_commands = self.state.config.verify.e2e.clone();
7612 let cache_dir = self.state.config.cache_dir();
7613 let context = format!("round {round}");
7614 let wait = self.repository_wait();
7615 let (e2e, verify_retried) = with_cache_lease(
7616 &mut self.state,
7617 cache_dir.as_deref(),
7618 "e2e",
7619 "e2e",
7620 &winner.worktree,
7621 &head,
7622 verify_timeout,
7623 &context,
7624 &wait,
7625 |state, budget| {
7626 let shell = shell.clone();
7627 let e2e_commands = e2e_commands.clone();
7628 let worktree = winner.worktree.clone();
7629 let context = context.clone();
7630 async move {
7631 run_e2e_with_retry(
7632 state,
7633 &shell,
7634 &e2e_commands,
7635 &worktree,
7636 budget,
7637 &context,
7638 )
7639 .await
7640 }
7641 },
7642 )
7643 .await?;
7644 (e2e, verify_retried, false, None)
7645 };
7646
7647 let expected = records.len();
7648 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7649 let incomplete = answered < expected;
7650 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7651 let policy = self.state.config.graph.incomplete_review;
7652 let clean = round_is_clean(
7653 blocking,
7654 e2e_ok,
7655 answered,
7656 expected,
7657 round_quota_missing,
7658 policy,
7659 );
7660
7661 let mut round_record = ReviewRound {
7662 round,
7663 head: head.clone(),
7664 verified_head: None,
7665 verified_at: None,
7666 reviews: records,
7667 e2e,
7668 verify_retried,
7669 e2e_deferred,
7670 e2e_defer_reason,
7671 fix: None,
7672 blocking,
7673 answered,
7674 expected,
7675 clean,
7676 progressed: false,
7677 vote_split,
7678 reconsideration,
7679 verdict: None,
7680 };
7681 round_record.verdict = ReviewVote::worst(
7685 round_record
7686 .final_votes()
7687 .into_iter()
7688 .map(|(_, _, vote)| vote),
7689 );
7690 if !matches!(
7701 round_record.e2e_status(),
7702 E2eStatus::Deferred | E2eStatus::NotConfigured
7703 ) {
7704 round_record.verified_head = Some(head.clone());
7705 round_record.verified_at = Some(Timestamp::now());
7706 }
7707
7708 if incomplete {
7709 let missing: Vec<String> = round_record
7710 .reviews
7711 .iter()
7712 .filter(|r| r.failed.is_some())
7713 .map(|r| format!("review-{}", r.reviewer))
7714 .collect();
7715 self.state.event(
7716 "review",
7717 format!(
7718 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7719 missing.join(", ")
7720 ),
7721 );
7722 }
7723
7724 if clean {
7725 self.state.event(
7726 "review",
7727 if incomplete && policy == IncompleteReviewPolicy::Warn {
7728 format!(
7729 "round {round}: clean (warn policy, incomplete panel) — no \
7730 blocking findings from the seats that answered, verification green"
7731 )
7732 } else if incomplete {
7733 format!(
7734 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7735 quorum) — no blocking findings from the seats that answered, \
7736 verification green",
7737 expected - answered
7738 )
7739 } else {
7740 format!("round {round}: clean — no blocking findings, verification green")
7741 },
7742 );
7743 self.state.reviews.push(round_record);
7744 self.state.status = RunStatus::Gating;
7745 self.state.save()?;
7746 return Ok(());
7747 }
7748
7749 if incomplete && blocking == 0 && e2e_ok {
7757 self.state.reviews.push(round_record);
7758 self.state.save()?;
7759 if round == max_rounds {
7760 self.state.status = RunStatus::Blocked;
7761 self.state.event(
7762 "review",
7763 format!(
7764 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7765 refusing to call it clean",
7766 expected - answered
7767 ),
7768 );
7769 return Ok(());
7770 }
7771 continue;
7772 }
7773
7774 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7786 self.state.reviews.push(round_record);
7787 return self
7788 .stop_unresolved_review(
7789 "the round's own verification could not run",
7790 &shell,
7791 &winner.worktree,
7792 )
7793 .await;
7794 }
7795
7796 if round == max_rounds {
7797 self.state.reviews.push(round_record);
7798 return self
7799 .stop_reviewing(
7800 &format!(
7801 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7802 ),
7803 &shell,
7804 &winner.worktree,
7805 )
7806 .await;
7807 }
7808
7809 if !self.fix_review_round(&winner, round_record).await? {
7810 return Ok(());
7811 }
7812 }
7813 Ok(())
7814 }
7815
7816 async fn fix_review_round(
7819 &mut self,
7820 winner: &Candidate,
7821 mut round_record: ReviewRound,
7822 ) -> Result<bool> {
7823 let round = round_record.round;
7824 let shell = self.state.config.shell();
7825 let step = self.run_fix_round(winner, &mut round_record).await?;
7826 if matches!(step, FixStep::Resource) {
7827 self.stop_for_resource()?;
7828 return Ok(false);
7829 }
7830 if self.state.reviews.len() < round {
7831 self.state.reviews.push(round_record);
7832 } else {
7833 self.state.reviews[round - 1] = round_record;
7834 }
7835 self.state.save()?;
7836
7837 if matches!(step, FixStep::Recorded { report_lost: true }) {
7850 self.stop_reviewing(
7851 "the fixer's adoption report never came back, even after resuming its \
7852 own seat; refusing to start another round against the same worktree \
7853 while that is unresolved",
7854 &shell,
7855 &winner.worktree,
7856 )
7857 .await?;
7858 return Ok(false);
7859 }
7860
7861 let streak = self
7862 .state
7863 .reviews
7864 .iter()
7865 .rev()
7866 .take_while(|r| !r.progressed)
7867 .count();
7868 if streak >= STAGNANT_LIMIT {
7869 self.stop_reviewing(
7870 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7871 &shell,
7872 &winner.worktree,
7873 )
7874 .await?;
7875 return Ok(false);
7876 }
7877 Ok(true)
7878 }
7879
7880 async fn run_fix_round(
7882 &mut self,
7883 winner: &Candidate,
7884 round_record: &mut ReviewRound,
7885 ) -> Result<FixStep> {
7886 let round = round_record.round;
7887 let max_rounds = self.state.config.graph.review_rounds;
7888 let run_id = self.state.id.clone();
7889 let prompts = self.state.config.prompts.clone();
7890 let language = self.state.config.graph.language.clone();
7891 let sessions = self.state.config.graph.sessions;
7892 let artifacts = self.merge_repair_artifacts();
7893 let base = self.landing_base();
7894 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7895 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7896 let this_round_verification = round_record.verification_summary(&head);
7897 let all_findings: Vec<_> = round_record
7898 .reviews
7899 .iter()
7900 .flat_map(|r| r.findings.iter().cloned())
7901 .collect();
7902 let blocking_findings: Vec<_> = all_findings
7905 .iter()
7906 .filter(|f| f.severity.blocks())
7907 .cloned()
7908 .collect();
7909 let fix_prompt = prompt::fix_in(
7910 &self.state.instruction,
7911 &blocking_findings,
7912 this_round_verification.as_ref(),
7913 round,
7914 max_rounds,
7915 &language,
7916 &self.state.config.graph.github_language,
7917 );
7918 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7919 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7920 let (job, seat, out) = self
7921 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7922 prompt: fix_prompt.clone(),
7923 spec,
7924 seat,
7925 cwd: winner.worktree.clone(),
7926 timeout,
7927 allow_write: true,
7928 sessions,
7929 artifacts: artifacts.clone(),
7930 stem: format!("fix-{round}"),
7931 handover: None,
7932 })
7933 .await;
7934 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7937 return Ok(FixStep::Resource);
7938 }
7939 let agent_id = seat.agent.clone();
7940
7941 let mut fix = FixRecord {
7942 agent: agent_id,
7943 addressed: Vec::new(),
7944 rejected: Vec::new(),
7945 notes: String::new(),
7946 committed: false,
7947 failed: None,
7948 duration_ms: 0,
7949 continuation: None,
7950 };
7951 let mut continuation = ContinuationRecord::not_needed();
7952 let mut final_seat = seat.clone();
7953 match out {
7954 AgentOutcome::Ok(o) => {
7955 fix.duration_ms = o.duration_ms;
7956 let parsed = verdict::extract_json::<FixReport>(&o.text);
7957 let incomplete_reason = match &parsed {
7964 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7965 "the reply parsed, but it reported a command whose own CLI never \
7966 confirmed an exit status"
7967 .to_owned(),
7968 ),
7969 Ok(_) => None,
7970 Err(e) => Some(e.to_string()),
7971 };
7972 match incomplete_reason {
7973 None => {
7974 let report = parsed.expect("checked Ok above");
7975 fix.addressed = report.addressed;
7976 fix.rejected = report.rejected;
7977 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7978 }
7979 Some(reason) => {
7980 let (resumed_seat, resolved, failure, cont) = self
7981 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7982 .await;
7983 fix.duration_ms += cont.cumulative_wait_ms;
7984 continuation = cont;
7985 final_seat = resumed_seat;
7986 match resolved {
7987 Some(report) => {
7988 fix.addressed = report.addressed;
7989 fix.rejected = report.rejected;
7990 fix.notes =
7991 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7992 }
7993 None => fix.failed = failure,
7994 }
7995 }
7996 }
7997 }
7998 AgentOutcome::Dropped(o) => {
8000 fix.duration_ms = o.duration_ms;
8001 let why = o
8002 .dropped
8003 .as_ref()
8004 .map(|d| d.why.as_str())
8005 .unwrap_or("the CLI ended the stream without delivering its answer");
8006 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8007 }
8008 AgentOutcome::Quota(o) => {
8009 self.state.quota.push(QuotaLoss::from_report(
8010 final_seat.key.clone(),
8011 "fix",
8012 o.quota.as_ref(),
8013 ));
8014 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8015 }
8016 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8017 }
8018 if self.state.resource_block.is_some() {
8019 self.state.seats.insert(final_seat.key.clone(), final_seat);
8020 return Ok(FixStep::Resource);
8021 }
8022 fix.continuation = Some(continuation);
8023 self.state.seats.insert(final_seat.key.clone(), final_seat);
8024 let coordination = self.coordinate(&winner.worktree).await?;
8025 if let Ok(r) = git::rescue_commit(
8026 &winner.worktree,
8027 &format!("magi: review round {round} fixes (uncommitted work)"),
8028 )
8029 .await
8030 {
8031 self.state.note_withheld("fix", &r.withheld);
8032 }
8033 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8034 fix.committed = after != before;
8035 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8043 drop(coordination);
8044 let progressed = diff_after != patch;
8045 let commit_note = if fix.committed {
8046 "committed"
8047 } else {
8048 "NO new commit"
8049 };
8050 let tree_note = if progressed {
8051 "changed vs base"
8052 } else {
8053 "unchanged vs base"
8054 };
8055 self.state.event(
8056 "fix",
8057 match &fix.failed {
8058 Some(reason) => {
8064 format!(
8065 "round {round}: fixer's adoption report was lost ({reason}); \
8066 {commit_note}, tree {tree_note}"
8067 )
8068 }
8069 None => format!(
8070 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8071 {tree_note}{}",
8072 fix.addressed.len(),
8073 fix.rejected.len(),
8074 if continuation.outcome == ContinuationOutcome::Resumed {
8075 format!(
8076 " (adoption report recovered after {} continuation(s))",
8077 continuation.attempts
8078 )
8079 } else {
8080 String::new()
8081 },
8082 ),
8083 },
8084 );
8085 round_record.fix = Some(fix);
8086 round_record.progressed = progressed;
8087 Ok(FixStep::Recorded {
8088 report_lost: matches!(
8089 continuation.outcome,
8090 ContinuationOutcome::Exhausted
8091 | ContinuationOutcome::QuotaLost
8092 | ContinuationOutcome::NoSession
8093 ),
8094 })
8095 }
8096
8097 fn record_contested_handoff(&mut self) {
8131 if self.state.contested_handoff.is_some() {
8132 return;
8133 }
8134 let Some(contested) = self
8135 .state
8136 .reviews
8137 .last()
8138 .and_then(ReviewRound::contested_handoff)
8139 else {
8140 return;
8141 };
8142 self.state.event(
8143 "review",
8144 format!(
8145 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8146 wait for the owner's approval",
8147 contested.findings.len(),
8148 contested.rejecters.len()
8149 ),
8150 );
8151 self.state.contested_handoff = Some(contested);
8152 }
8153
8154 async fn stop_unresolved_review(
8155 &mut self,
8156 why: &str,
8157 shell: &[String],
8158 worktree: &Path,
8159 ) -> Result<()> {
8160 self.stop_reviewing_inner(why, shell, worktree, true).await
8161 }
8162
8163 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8164 self.stop_reviewing_inner(why, shell, worktree, false).await
8165 }
8166
8167 async fn stop_reviewing_inner(
8168 &mut self,
8169 why: &str,
8170 shell: &[String],
8171 worktree: &Path,
8172 repairable: bool,
8173 ) -> Result<()> {
8174 let round_idx = self.state.reviews.len() - 1;
8175 let needs_catchup_run = matches!(
8183 self.state.reviews[round_idx].e2e_status(),
8184 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8185 );
8186 if needs_catchup_run {
8187 let round = self.state.reviews[round_idx].round;
8188 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8189 let commands = self.state.config.verify.e2e.clone();
8190 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8191 let cache_dir = self.state.config.cache_dir();
8192 let context = format!(
8193 "round {round}: verification unresolved, catching up before the final decision"
8194 );
8195 let wait = self.repository_wait();
8196 let (outcomes, verify_retried) = with_cache_lease(
8197 &mut self.state,
8198 cache_dir.as_deref(),
8199 "e2e",
8200 "e2e",
8201 worktree,
8202 &attempted_head,
8203 timeout,
8204 &context,
8205 &wait,
8206 |state, budget| {
8207 let shell = shell.to_vec();
8208 let commands = commands.clone();
8209 let context = context.clone();
8210 async move {
8211 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8212 .await
8213 }
8214 },
8215 )
8216 .await?;
8217 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8218 self.state.record_base_progress(&attempted_head);
8219 }
8220 let last = &mut self.state.reviews[round_idx];
8221 last.e2e = outcomes;
8222 last.verify_retried = verify_retried;
8223 last.verified_head = Some(attempted_head);
8230 last.verified_at = Some(Timestamp::now());
8231 if verify_inconclusive(&last.e2e) {
8232 self.state.save()?;
8239 return Ok(());
8240 }
8241 last.e2e_deferred = false;
8242 if last.answered == last.expected {
8247 last.clean = round_is_clean(
8248 last.blocking,
8249 last.e2e.iter().all(CommandOutcome::ok),
8250 last.answered,
8251 last.expected,
8252 0,
8253 self.state.config.graph.incomplete_review,
8254 );
8255 }
8256 }
8257 let last = &self.state.reviews[round_idx];
8258 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8259
8260 match last.e2e_status() {
8261 E2eStatus::Failed => {
8262 let red: Vec<String> = last
8263 .e2e
8264 .iter()
8265 .filter(|o| !o.ok())
8266 .map(|o| {
8267 format!(
8268 "`{}` -> {:?}\n{}",
8269 o.command,
8270 o.code,
8271 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8272 )
8273 })
8274 .collect();
8275 let can_repair = last.blocking == 0 && last.fix.is_none();
8276 self.state.reviews[round_idx].clean = false;
8277 self.state
8278 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8279 self.state.status = RunStatus::Blocked;
8280 if repairable
8281 && can_repair
8282 && self.state.reviews.len() < self.state.config.graph.review_rounds
8283 {
8284 self.state.status = RunStatus::Reviewing;
8285 }
8286 }
8287 E2eStatus::ResourceBlocked => {
8292 self.state.event(
8293 "review",
8294 format!(
8295 "{why}; e2e could not run (resource unavailable); not \
8296 deciding yet"
8297 ),
8298 );
8299 }
8300 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8301 self.state.event(
8302 "review",
8303 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8304 );
8305 self.record_contested_handoff();
8306 self.state.status = RunStatus::Gating;
8307 }
8308 }
8309 self.state.save()?;
8310 Ok(())
8311 }
8312
8313 async fn gate(&mut self) -> Result<()> {
8316 if self.hold_base_sync().await? {
8317 return Ok(());
8318 }
8319 if self.state.status == RunStatus::Failed
8331 || self
8332 .state
8333 .base_sync
8334 .as_ref()
8335 .is_some_and(|s| s.conflict.is_some())
8336 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8337 != Some(RunStatus::Gating)
8338 {
8339 return Ok(());
8340 }
8341 let Some(winner) = self.state.winner().cloned() else {
8342 return Ok(());
8343 };
8344 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8345 self.state.reconcile_verification_head(&head);
8346 self.state.save()?;
8347 let wait = self.repository_wait();
8348 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8349 return Ok(());
8350 }
8351 if self.state.gate_ran {
8352 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8363 self.state.status = RunStatus::Blocked;
8364 self.state.save()?;
8365 }
8366 return Ok(());
8367 }
8368 self.state.status = RunStatus::Gating;
8369 let mut outcomes = self.run_gate(&winner).await?;
8370 let mut attempted_head = self.state.gate_head.clone();
8371 loop {
8372 if verify_inconclusive(&outcomes) {
8383 self.state.save()?;
8384 return Ok(());
8385 }
8386 if outcomes.iter().all(CommandOutcome::ok) {
8387 break;
8388 }
8389 let fixes_before = self.state.gate_fixes.len();
8390 match self.gate_fix_round(&winner, &outcomes).await? {
8391 GateFix::Retry => {
8392 outcomes = self.run_gate(&winner).await?;
8393 attempted_head = self.state.gate_head.clone();
8394 }
8395 GateFix::Stop => {
8396 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8397 if self.state.gate_fixes.len() > fixes_before
8398 && attempted_head.as_deref() != Some(current.as_str())
8399 && verify_current_head(&mut self.state, &winner, &wait).await?
8400 {
8401 outcomes = self.run_gate(&winner).await?;
8404 attempted_head = self.state.gate_head.clone();
8405 } else {
8406 break;
8407 }
8408 }
8409 GateFix::Defer => {
8410 self.state.save()?;
8411 return Ok(());
8412 }
8413 }
8414 }
8415 let passed = outcomes.iter().all(CommandOutcome::ok);
8416 self.state.gate = outcomes;
8417 self.state.gate_ran = true;
8418 self.state.gate_head = attempted_head;
8419 if !self.state.gate.is_empty()
8420 && !verify_inconclusive(&self.state.gate)
8421 && let Some(head) = self.state.gate_head.clone()
8422 {
8423 self.state.record_base_progress(&head);
8424 }
8425 self.state.save()?;
8426 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8427 return Ok(());
8428 }
8429 if !passed {
8430 self.state.status = RunStatus::Blocked;
8431 let spent = self.state.gate_fixes.len();
8432 self.state.event(
8433 "gate",
8434 if spent == 0 {
8435 "gate failed; not merging".to_owned()
8436 } else {
8437 format!("gate failed after {spent} gate-fix round(s); not merging")
8438 },
8439 );
8440 }
8441 self.state.save()?;
8442 Ok(())
8443 }
8444
8445 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8455 let commands = state.config.verify.pre_gate.clone();
8456 if commands.is_empty() {
8457 return;
8458 }
8459 let shell = state.config.shell();
8460 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8461 let (outcomes, _) = run_commands(
8462 state,
8463 "pre_gate",
8464 "pre_gate",
8465 0,
8466 &shell,
8467 &commands,
8468 &winner.worktree,
8469 timeout,
8470 )
8471 .await;
8472 for o in &outcomes {
8473 if !o.ok() {
8474 tracing::warn!(
8475 "pre_gate `{}` failed ({:?}); the gate decides",
8476 o.command,
8477 o.code
8478 );
8479 }
8480 state.event(
8481 "pre_gate",
8482 format!(
8483 "`{}` -> {}",
8484 o.command,
8485 if o.ok() {
8486 "pass".to_owned()
8487 } else {
8488 format!(
8489 "FAIL ({:?})\n{}",
8490 o.code,
8491 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8492 )
8493 }
8494 ),
8495 );
8496 }
8497 state.pre_gate = outcomes;
8498 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8499 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8500 Ok(head) => {
8501 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8502 state.pre_gate_commit = Some(head);
8503 }
8504 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8505 },
8506 Ok(false) => {}
8507 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8508 }
8509 if let Err(e) = state.save() {
8510 tracing::warn!("could not persist the pre_gate record: {e:#}");
8511 }
8512 }
8513
8514 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8517 let wait = self.repository_wait();
8518 run_landing_gate(&mut self.state, winner, &wait).await
8519 }
8520
8521 async fn gate_fix_round(
8534 &mut self,
8535 winner: &Candidate,
8536 outcomes: &[CommandOutcome],
8537 ) -> Result<GateFix> {
8538 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8539 g.phase == crate::repair_budget::Phase::Fixing
8540 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8541 }) {
8542 self.state.gate_fixes.len().saturating_add(1)
8543 } else {
8544 self.state.config.graph.gate_fix_rounds
8545 };
8546 let spent = self.state.gate_fixes.len();
8547 if spent >= cap {
8548 if cap > 0 {
8549 self.state.event(
8550 "gate",
8551 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8552 );
8553 }
8554 return Ok(GateFix::Stop);
8555 }
8556 if !gate_fixable(outcomes) {
8557 self.state.event(
8558 "gate",
8559 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8560 command or similar); not spending a fix round on it",
8561 );
8562 return Ok(GateFix::Stop);
8563 }
8564 let min_free = self.state.config.disk.min_free_bytes;
8565 if min_free > 0 {
8566 match crate::disk::free_bytes(&winner.worktree) {
8567 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8568 Ok(free) => {
8569 self.state.event(
8570 "gate",
8571 format!(
8572 "only {free} bytes free ({min_free} required by `[disk] \
8573 min_free_bytes`); not spending a fix round on a failure the disk \
8574 may explain"
8575 ),
8576 );
8577 return Ok(GateFix::Stop);
8578 }
8579 Err(e) => {
8580 self.state.event(
8581 "gate",
8582 format!("free disk space could not be measured ({e:#}); no fix round"),
8583 );
8584 return Ok(GateFix::Stop);
8585 }
8586 }
8587 }
8588
8589 let attempt = spent + 1;
8590 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8591 let base = self.landing_base();
8592 let fix_prompt = prompt::gate_fix_in(
8593 &self.state.instruction,
8594 &failed,
8595 attempt,
8596 cap,
8597 &self.state.config.graph.language,
8598 &self.state.config.graph.github_language,
8599 );
8600 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8601 let sessions = self.state.config.graph.sessions;
8602 let artifacts = self.merge_repair_artifacts();
8603 self.state.event(
8604 "gate",
8605 format!("gate failed; gate-fix round {attempt} of {cap}"),
8606 );
8607 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8608 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8609 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8612 head: before.clone(),
8613 passed: false,
8614 outcomes: Vec::new(),
8615 });
8616 self.state.save()?;
8617 let (_, seat, out) = self
8618 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8619 prompt: fix_prompt.clone(),
8620 spec,
8621 seat,
8622 cwd: winner.worktree.clone(),
8623 timeout,
8624 allow_write: true,
8625 sessions,
8626 artifacts: artifacts.clone(),
8627 stem: format!("gate-fix-{attempt}"),
8628 handover: None,
8629 })
8630 .await;
8631 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8634 return Ok(GateFix::Defer);
8635 }
8636 let mut record = GateFixRecord {
8637 agent: seat.agent.clone(),
8638 failed,
8639 notes: String::new(),
8640 committed: false,
8641 error: None,
8642 };
8643 match out {
8644 AgentOutcome::Ok(o) => {
8645 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8648 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8649 }
8650 }
8651 AgentOutcome::Dropped(_) => {
8652 record.error = Some("the CLI dropped the stream".to_owned());
8653 }
8654 AgentOutcome::Quota(o) => {
8655 self.state.quota.push(QuotaLoss::from_report(
8656 seat.key.clone(),
8657 "gate-fix",
8658 o.quota.as_ref(),
8659 ));
8660 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8661 }
8662 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8663 }
8664 self.state.seats.insert(seat.key.clone(), seat);
8665 let coordination = self.coordinate(&winner.worktree).await?;
8666 if let Ok(r) = git::rescue_commit(
8667 &winner.worktree,
8668 &format!("magi: gate fix {attempt} (uncommitted work)"),
8669 )
8670 .await
8671 {
8672 self.state.note_withheld("gate-fix", &r.withheld);
8673 }
8674 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8675 record.committed = after != before;
8676 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8677 let note = record.error.clone();
8678 self.state.gate_fixes.push(record);
8679 self.state.save()?;
8680 if !changed {
8681 self.state.event(
8682 "gate",
8683 match note {
8684 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8685 None => format!("gate-fix round {attempt}: the tree did not change"),
8686 },
8687 );
8688 return Ok(GateFix::Stop);
8689 }
8690 self.state.event(
8691 "gate",
8692 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8693 );
8694
8695 drop(coordination);
8696 let wait = self.repository_wait();
8697 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8698 }
8699
8700 async fn guarded_pr_message(
8707 &mut self,
8708 winner: &Candidate,
8709 facts: Option<&BranchFacts>,
8710 posting: bool,
8711 ) -> Result<Option<PrMessage>> {
8712 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8713 let head = match final_head.as_deref() {
8714 Some(h) => crate::pr_outcome::FinalHead::At(h),
8715 None => crate::pr_outcome::FinalHead::Unknown,
8716 };
8717 let summarized = if posting {
8719 Some(self.summarize_change(winner, final_head.as_deref()).await)
8720 } else {
8721 None
8722 };
8723 if self.state.resource_block.is_some() {
8728 self.state.event(
8729 "pr-summary",
8730 "a shared resource was busy; nothing was published, resume to summarize again",
8731 );
8732 return Ok(None);
8733 }
8734 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8735 anyhow::bail!(
8736 "the final commit moved during public description generation; retry before publishing"
8737 );
8738 }
8739 let note = match (&summarized, &self.state.change_summary) {
8740 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8741 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8742 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8743 (None, _) => ChangeNote::NotAsked,
8744 };
8745 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8746 if !posting {
8747 let identity = crate::scrub::Identity::current();
8748 return Ok(Some(PrMessage {
8749 title: crate::scrub::scrub(&pr.title, &identity),
8750 body: crate::scrub::scrub(&pr.body, &identity),
8751 }));
8752 }
8753 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8756 if let Some(known) = self
8757 .state
8758 .github_text
8759 .clone()
8760 .filter(|g| g.fingerprint == raw_fp)
8761 {
8762 return self.settle_withheld(winner, &pr, known).await;
8763 }
8764 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8765 let mut violations =
8766 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8767 let identity = crate::scrub::Identity::current();
8768 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8769 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8770 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8771 {
8772 violations.push(crate::github_text::Violation::SensitiveData);
8773 }
8774 let has_language = violations
8778 .iter()
8779 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8780 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8781 self.state.event(
8782 "github-text",
8783 if has_language {
8784 format!("description rejected: {violations:?}; requesting one rewrite")
8785 } else {
8786 format!("description rejected: {violations:?}; asking the owner")
8787 },
8788 );
8789 let mut rewritten = false;
8790 let mut rejected_rewrite = None;
8791 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8792 {
8793 let output = text;
8794 if let Some(title) = summary_title(&output) {
8795 let raw_body = summary_without_title(&output);
8798 let clean_title = crate::scrub::scrub(&title, &identity);
8799 let mut body = crate::scrub::scrub(&raw_body, &identity);
8800 let redacted = clean_title != title || body != raw_body;
8801 let title = clean_title;
8802 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8803 if !complete {
8804 self.state.event(
8805 "github-text",
8806 "description rewrite omitted required findings",
8807 );
8808 }
8809 if !body.trim().is_empty() {
8810 body.push_str("\n\n");
8811 body.push_str(&public_pr_evidence(&self.state, head));
8812 }
8813 let marker = format!("magi:run/{}", self.state.id);
8814 if !body.trim().is_empty() && !body.contains(&marker) {
8815 body.push_str(&format!("\n\n{marker}"));
8816 }
8817 let rewrite = PrMessage {
8818 title: title.clone(),
8819 body: body.clone(),
8820 };
8821 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8822 let mut failed = crate::github_text::check_for(
8823 &self.state.config.graph,
8824 &title,
8825 &body,
8826 redecision,
8827 );
8828 if (!crate::github_text::shareable(&title)
8829 || !crate::github_text::shareable(&body))
8830 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8831 {
8832 failed.push(crate::github_text::Violation::SensitiveData);
8833 }
8834 if complete
8835 && !title.trim().is_empty()
8836 && !body.trim().is_empty()
8837 && !failed.is_empty()
8838 {
8839 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8840 title: title.clone(),
8841 body: body.clone(),
8842 failed: failed
8843 .iter()
8844 .map(|v| crate::github_text::category(*v).to_owned())
8845 .collect(),
8846 });
8847 }
8848 if complete
8849 && !title.trim().is_empty()
8850 && !raw_body.trim().is_empty()
8851 && failed.is_empty()
8852 {
8853 decision = redecision;
8854 pr = PrMessage { title, body };
8855 rewritten = true;
8856 self.state
8857 .event("github-text", "description rewrite passed");
8858 if redacted {
8859 self.state.event(
8860 "github-text",
8861 "sensitive data removed from description rewrite",
8862 );
8863 }
8864 }
8865 }
8866 }
8867 if !rewritten {
8868 self.state.event(
8869 "github-text",
8870 "description rewrite unavailable or rejected; asking the owner",
8871 );
8872 if let Some(w) =
8873 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8874 {
8875 self.retire_github_text_question("the withheld text changed");
8878 let record = crate::run::GithubTextAsk {
8879 rewrite: rejected_rewrite,
8880 fingerprint: raw_fp,
8881 title: w.title,
8882 body: w.body,
8883 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8884 question: None,
8885 asks: 0,
8886 resolved: false,
8887 chosen_title: None,
8888 choice: None,
8889 publication: None,
8890 };
8891 return self.settle_withheld(winner, &pr, record).await;
8892 }
8893 }
8894 }
8895 let (title, body) =
8896 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8897 Ok(Some(PrMessage { title, body }))
8898 }
8899
8900 async fn summarize_change(
8911 &mut self,
8912 winner: &Candidate,
8913 head: Option<&str>,
8914 ) -> std::result::Result<(), String> {
8915 let Some(head) = head else {
8916 return Err("the final commit could not be read".to_owned());
8917 };
8918 let remote = self.state.config.merge.remote.clone();
8919 let tracking = format!("{remote}/{}", self.state.base_branch);
8920 if !git::rev_exists(&winner.worktree, &tracking).await {
8921 return Err(format!("the remote base `{tracking}` could not be read"));
8922 }
8923 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8924 Ok(b) if !b.is_empty() => b,
8925 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8926 };
8927 let evidence = crate::pr_outcome::summary_evidence(
8928 &self.state,
8929 crate::pr_outcome::FinalHead::At(head),
8930 );
8931 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8932 if self
8933 .state
8934 .change_summary
8935 .as_ref()
8936 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8937 {
8938 return Ok(());
8939 }
8940 if self.state.change_summary.take().is_some() {
8941 self.state.event(
8942 "pr-summary",
8943 "the final diff or evidence changed; the earlier change summary was discarded",
8944 );
8945 }
8946 if let Err(e) = write_artifact(
8948 &self.state,
8949 &format!("public-description-{fingerprint}.md"),
8950 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8951 ) {
8952 tracing::warn!("could not save public description evidence: {e:#}");
8953 }
8954 let chain = match agent::pick_chain(
8955 &self.state.config.agents,
8956 self.state.config.roles.synthesizer.as_ref(),
8957 &agent::installed,
8958 "synthesizer",
8959 ) {
8960 Ok(chain) => chain,
8961 Err(e) => {
8962 tracing::warn!("no agent for the change summary: {e:#}");
8963 self.state.event("pr-summary", "no summary agent available");
8964 return Err("no agent was available to summarize it".to_owned());
8965 }
8966 };
8967 let subjects = git::subjects(&winner.worktree, &base, head)
8968 .await
8969 .unwrap_or_default();
8970 let stat = git::diff_stat(&winner.worktree, &base, head)
8971 .await
8972 .unwrap_or_default();
8973 let stat: String = stat.chars().take(6_000).collect();
8974 let mut prompt = crate::prompt::change_summary_in(
8975 &base,
8976 head,
8977 &subjects,
8978 stat.trim(),
8979 &self.state.config.graph.github_language,
8980 );
8981 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8982 let sessions = self.state.config.graph.sessions;
8983 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8984 let prompts = self.state.config.prompts.clone();
8985 let cache = self.state.config.cache_dir();
8986 let run = self.state.id.clone();
8987 for (i, spec) in chain.into_iter().enumerate() {
8988 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8989 let github_spec = match agent::for_github_text(
8990 &spec,
8991 &self.state.config.graph.github_language,
8992 &winner.worktree,
8993 ) {
8994 Ok(spec) => spec,
8995 Err(e) => {
8996 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8997 continue;
8998 }
8999 };
9000 let job = SeatJob {
9001 spec: github_spec,
9002 seat,
9003 cwd: winner.worktree.clone(),
9004 prompt: prompt.clone(),
9005 timeout,
9006 allow_write: false,
9007 sessions,
9008 artifacts: agent::artifacts_dir(&self.state.dir()),
9009 stem: if i == 0 {
9010 "change-summary".to_owned()
9011 } else {
9012 format!("change-summary-{}", spec.id)
9013 },
9014 handover: None,
9015 };
9016 let ctx = WaveCtx {
9017 run: &run,
9018 node: "pr-summary",
9019 prompts: &prompts,
9020 cache: cache.as_deref(),
9021 round: None,
9022 carry_seats: false,
9023 pauses: Some((&self.pause, &self.interrupt)),
9024 };
9025 let (seat, outcome) =
9026 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9027 self.state.seats.insert(seat.key.clone(), seat);
9028 let text = match outcome {
9029 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9030 AgentOutcome::ResourceBlocked(why) => {
9031 if self.state.resource_block.is_none() {
9034 self.state.resource_block = Some(crate::run::ResourceBlock {
9035 cause: crate::run::ResourceBlockCause::Other,
9036 node: "pr-summary".to_owned(),
9037 kind: "shared resource".to_owned(),
9038 detail: why,
9039 at: Timestamp::now(),
9040 });
9041 }
9042 return Err("a shared resource was busy".to_owned());
9043 }
9044 _ => continue,
9045 };
9046 let blind = &self.state.config.blind;
9047 let Some(title) = summary_title(&text) else {
9048 continue;
9049 };
9050 let body: String = summary_without_title(&text)
9051 .lines()
9052 .filter(|l| !l.contains("magi:run/"))
9053 .collect::<Vec<_>>()
9054 .join("\n");
9055 let body = blind::sanitize_prose(body.trim(), blind);
9056 let title = blind::sanitize_prose(&title, blind);
9057 if body.trim().is_empty() || title.trim().is_empty() {
9058 continue;
9059 }
9060 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9061 continue;
9062 }
9063 if git::rev_parse(&winner.worktree, "HEAD")
9065 .await
9066 .ok()
9067 .as_deref()
9068 != Some(head)
9069 {
9070 return Err("the branch moved while the summary was written".to_owned());
9071 }
9072 if git::merge_base(&winner.worktree, &tracking, head)
9073 .await
9074 .ok()
9075 .as_deref()
9076 != Some(base.as_str())
9077 || crate::pr_outcome::summary_evidence(
9078 &self.state,
9079 crate::pr_outcome::FinalHead::At(head),
9080 ) != evidence
9081 {
9082 return Err("the summary evidence changed while it was written".to_owned());
9083 }
9084 self.state.change_summary = Some(crate::run::ChangeSummary {
9085 head: head.to_owned(),
9086 base,
9087 evidence: fingerprint,
9088 title,
9089 body,
9090 });
9091 self.state.event(
9092 "pr-summary",
9093 format!("change summary written by {} for the final diff", spec.id),
9094 );
9095 if let Err(e) = self.state.save() {
9096 tracing::warn!("could not save the change summary: {e:#}");
9097 }
9098 return Ok(());
9099 }
9100 self.state
9101 .event("pr-summary", "no agent produced a change summary");
9102 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9103 }
9104
9105 async fn rewrite_pr_text(
9115 &mut self,
9116 winner: &Candidate,
9117 pr: &PrMessage,
9118 violations: &[crate::github_text::Violation],
9119 ) -> Option<String> {
9120 let prompt = format!(
9121 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9122 pr.title, pr.body
9123 );
9124 let sessions = self.state.config.graph.sessions;
9125 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9126 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9129 let key = format!("impl-{}", winner.label);
9130 if let Some(spec) = self
9131 .state
9132 .config
9133 .agents
9134 .iter()
9135 .find(|a| a.id == winner.agent)
9136 .cloned()
9137 && let Some(existing) = self.state.seats.get(&key)
9138 && existing.agent == spec.id
9139 && agent::has_session(spec.kind, existing, sessions)
9140 {
9141 attempts.push((spec, Some(existing.clone())));
9142 }
9143 match agent::pick_chain(
9144 &self.state.config.agents,
9145 self.state.config.roles.language_judge.as_ref(),
9146 &agent::installed,
9147 "language_judge",
9148 ) {
9149 Ok(chain) => {
9150 for spec in chain {
9151 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9152 continue;
9153 }
9154 attempts.push((spec, None));
9155 }
9156 }
9157 Err(e) if attempts.is_empty() => {
9158 tracing::warn!("no agent for the description rewrite: {e:#}");
9159 self.state
9160 .event("github-text", "no rewrite agent available");
9161 return None;
9162 }
9163 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9164 }
9165 let prompts = self.state.config.prompts.clone();
9166 let cache = self.state.config.cache_dir();
9167 let run = self.state.id.clone();
9168 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9169 let resumed = resumed_seat.is_some();
9170 let seat = resumed_seat.unwrap_or_else(|| {
9171 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9172 });
9173 let github_spec = match agent::for_github_text(
9174 &spec,
9175 &self.state.config.graph.github_language,
9176 &winner.worktree,
9177 ) {
9178 Ok(spec) => spec,
9179 Err(e) => {
9180 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9181 continue;
9182 }
9183 };
9184 let job = SeatJob {
9185 spec: github_spec,
9186 seat,
9187 cwd: winner.worktree.clone(),
9188 prompt: prompt.clone(),
9189 timeout: retry_budget(timeout_full, resumed),
9190 allow_write: false,
9191 sessions,
9192 artifacts: agent::artifacts_dir(&self.state.dir()),
9193 stem: if i == 0 {
9194 "github-text-rewrite".to_owned()
9195 } else {
9196 format!("github-text-rewrite-{}", spec.id)
9197 },
9198 handover: None,
9199 };
9200 let ctx = WaveCtx {
9201 run: &run,
9202 node: "github-text",
9203 prompts: &prompts,
9204 cache: cache.as_deref(),
9205 round: None,
9206 carry_seats: false,
9207 pauses: Some((&self.pause, &self.interrupt)),
9208 };
9209 let (seat, outcome) =
9210 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9211 self.state.seats.insert(seat.key.clone(), seat);
9212 match outcome {
9213 AgentOutcome::Ok(o)
9214 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9215 {
9216 return Some(o.text);
9217 }
9218 AgentOutcome::ResourceBlocked(_) => return None,
9219 _ => {}
9220 }
9221 }
9222 None
9223 }
9224
9225 fn retire_github_text_question(&mut self, why: &str) {
9227 let Some(id) = self
9228 .state
9229 .github_text
9230 .as_ref()
9231 .and_then(|g| g.question.clone())
9232 else {
9233 return;
9234 };
9235 let _ = ask::Questions::open().update(&id, |q| {
9236 q.abandon(why);
9237 Ok(())
9238 });
9239 }
9240
9241 async fn settle_withheld(
9245 &mut self,
9246 winner: &Candidate,
9247 pr: &PrMessage,
9248 mut g: crate::run::GithubTextAsk,
9249 ) -> Result<Option<PrMessage>> {
9250 use crate::github_text as gt;
9251 let store = ask::Questions::open();
9252 let timeout = self.state.config.graph.answer_timeout;
9253 if !g.resolved {
9254 let mut standing = g
9255 .question
9256 .as_ref()
9257 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9258 if let Some(q) = standing.as_ref()
9259 && q.status.open()
9260 && gt::expired(q, timeout)
9261 {
9262 let why = format!("no answer within {}s of asking", timeout.max(1));
9263 standing = store
9264 .update(&q.id, |q| {
9265 q.abandon(&why);
9266 Ok(())
9267 })
9268 .ok()
9269 .map(|(q, ())| q);
9270 }
9271 let mut ask_again: Option<Option<Vec<String>>> = None;
9272 match standing {
9273 None if g.asks <= 1 => {
9276 g.asks = 0;
9277 ask_again = Some(None);
9278 }
9279 None => {
9280 g.resolved = true;
9281 g.chosen_title = None;
9282 g.choice = Some(GithubTextChoice::Fallback);
9283 }
9284 Some(q) if q.status.open() => {
9285 self.park_on_github_text(&g)?;
9286 return Ok(None);
9287 }
9288 Some(q) => match gt::read_reply(&q) {
9289 gt::Reply::Fallback => {
9290 g.resolved = true;
9291 g.chosen_title = None;
9292 g.choice = Some(GithubTextChoice::Fallback);
9293 }
9294 gt::Reply::Original => {
9295 g.resolved = true;
9296 g.chosen_title = None;
9297 g.choice = Some(GithubTextChoice::Original);
9298 }
9299 gt::Reply::Rewrite => {
9300 g.resolved = true;
9301 g.chosen_title = None;
9302 g.choice = Some(if g.rewrite.is_some() {
9303 GithubTextChoice::Rewrite
9304 } else {
9305 GithubTextChoice::Fallback
9306 });
9307 }
9308 gt::Reply::Title(text) => {
9309 let first = text
9310 .lines()
9311 .map(str::trim)
9312 .find(|l| !l.is_empty())
9313 .unwrap_or("");
9314 let decision = if first.is_empty() {
9315 None
9316 } else {
9317 let candidate = PrMessage {
9318 title: first.to_owned(),
9319 body: String::new(),
9320 };
9321 self.judge_pr_language(&winner.worktree, &candidate).await
9322 };
9323 match gt::vet_title(
9324 &text,
9325 decision,
9326 self.state.config.graph.github_english(),
9327 ) {
9328 Ok(title) => {
9329 g.resolved = true;
9330 g.chosen_title = Some(title);
9331 g.choice = Some(GithubTextChoice::MyTitle);
9332 }
9333 Err(cats) if g.asks < 2 => {
9334 ask_again = Some(Some(
9335 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9336 ))
9337 }
9338 Err(_) => {
9339 g.resolved = true;
9340 g.chosen_title = None;
9341 g.choice = Some(GithubTextChoice::Fallback);
9342 }
9343 }
9344 }
9345 },
9346 }
9347 if let Some(rejected) = ask_again {
9348 let w = gt::Withheld {
9351 title: g.title,
9352 body: g.body,
9353 categories: g.categories.clone(),
9354 };
9355 let artifact = match crate::run::write_artifact(
9358 &self.state,
9359 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9360 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9361 ) {
9362 Ok(p) => Some(p),
9363 Err(e) => {
9364 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9365 None
9366 }
9367 };
9368 let (detail, comparison) = gt::question_content(
9369 &w,
9370 &pr.title,
9371 &pr.body,
9372 &self.state.id,
9373 rejected.as_deref(),
9374 artifact.as_deref(),
9375 g.rewrite.as_ref(),
9376 );
9377 let mut q = ask::Question::new(
9378 self.state.id.clone(),
9379 gt::ASK_NODE.to_owned(),
9380 gt::ASK_SEAT.to_owned(),
9381 gt::question_summary(&self.state.config.graph.language, &w),
9382 detail,
9383 gt::question_choices(&w, g.rewrite.as_ref()),
9384 );
9385 q.posting_comparison = Some(comparison);
9386 q.answer_timeout = timeout;
9387 g.question = Some(q.id.clone());
9390 g.asks += 1;
9391 self.state.github_text = Some(g.clone());
9392 self.state.event(
9393 "github-text",
9394 format!("asking the owner about the withheld text ({})", q.short()),
9395 );
9396 self.state.save()?;
9397 store
9398 .put_executable(&mut q)
9399 .context("file the posting-gate question")?;
9400 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9401 tracing::warn!(
9402 "could not notify about posting-gate question {}: {e:#}",
9403 q.short()
9404 );
9405 }
9406 self.park_on_github_text(&g)?;
9407 return Ok(None);
9408 }
9409 }
9410 let resolved = g.chosen_title.clone();
9411 let original = g.choice == Some(GithubTextChoice::Original);
9412 self.state.event(
9414 "github-text",
9415 if original {
9416 format!(
9417 "posting the original text for: {}",
9418 match (g.title, g.body) {
9419 (true, true) => "title, description",
9420 (true, false) => "title",
9421 _ => "description",
9422 }
9423 )
9424 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9425 "posting the saved rewrite title and description".to_owned()
9426 } else if resolved.is_some() {
9427 "posting the owner's replacement title".to_owned()
9428 } else {
9429 "posting the neutral text for the withheld fields".to_owned()
9430 },
9431 );
9432 self.state.github_text = Some(g.clone());
9433 self.state.save()?;
9434 if g.choice == Some(GithubTextChoice::Rewrite)
9435 && let Some(r) = g.rewrite.as_ref()
9436 {
9437 return Ok(Some(PrMessage {
9438 title: r.title.clone(),
9439 body: r.body.clone(),
9440 }));
9441 }
9442 let w = gt::Withheld {
9443 title: g.title,
9444 body: g.body,
9445 categories: g.categories.clone(),
9446 };
9447 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9448 if let Some(t) = resolved {
9449 title = t;
9450 }
9451 let english = gt::LanguageDecision {
9453 title_english: true,
9454 body_english: true,
9455 body_complete: true,
9456 };
9457 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9458 if original {
9459 if g.title {
9462 title = pr.title.clone();
9463 }
9464 if g.body {
9465 body = pr.body.clone();
9466 }
9467 }
9468 Ok(Some(PrMessage { title, body }))
9469 }
9470
9471 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9472 self.state.github_text = Some(g.clone());
9473 self.state.end_pass_parked(ParkCause::Question);
9474 self.state.parked = true;
9475 self.state.event(
9476 "github-text",
9477 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9478 );
9479 self.state.save()
9480 }
9481
9482 async fn judge_pr_language(
9485 &mut self,
9486 cwd: &Path,
9487 pr: &PrMessage,
9488 ) -> Option<crate::github_text::LanguageDecision> {
9489 self.state.config.roles.language_judge.as_ref()?;
9490 if !self.state.config.graph.language_gate() {
9491 return None;
9492 }
9493 let decision =
9494 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9495 self.state.event(
9496 "github-text",
9497 if decision.is_some() {
9498 "language decided by the language judge"
9499 } else {
9500 "language judge unavailable; using built-in heuristics"
9501 },
9502 );
9503 decision
9504 }
9505
9506 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9507 Box::pin(self.merge_inner())
9508 }
9509
9510 async fn merge_inner(&mut self) -> Result<()> {
9511 if self.resource_halt()? || self.hold_base_sync().await? {
9512 return Ok(());
9513 }
9514 if self
9529 .state
9530 .base_sync
9531 .as_ref()
9532 .is_some_and(|s| s.conflict.is_some())
9533 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9534 != Some(RunStatus::Gating)
9535 || !self.state.gate_allows_progress()
9543 {
9544 return Ok(());
9545 }
9546 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9555 return Ok(());
9556 }
9557 let Some(winner) = self.state.winner().cloned() else {
9558 return Ok(());
9559 };
9560 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9561 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9562 self.state.status = RunStatus::Blocked;
9563 self.state.event(
9564 "gate",
9565 "winner changed after gate; fresh verification required",
9566 );
9567 self.state.save()?;
9568 return Ok(());
9569 }
9570 let wait = self.repository_wait();
9571 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9572 return Ok(());
9573 }
9574 let repo = self.state.repo.clone();
9575 let base = self.state.base_branch.clone();
9576 let mode = self.state.config.merge.mode;
9577 let style = self.state.config.merge.style;
9578 let facts = if is_review_run(&self.state) {
9579 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9580 let start = review_base(
9581 &repo,
9582 &self.state.config.merge.remote,
9583 &base,
9584 &self.state.base_commit,
9585 &winner.branch,
9586 )
9587 .await;
9588 branch_facts(&repo, &start, &winner.branch).await
9589 } else {
9590 None
9591 };
9592 let leaked = if is_review_run(&self.state) {
9595 leaked_subjects(&self.state, &winner.branch).await
9596 } else {
9597 Some(Vec::new())
9598 };
9599 let Some(pr) = self
9603 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9604 .await?
9605 else {
9606 return Ok(());
9607 };
9608 let message = pr.commit_message();
9609
9610 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9611 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9612 return Ok(());
9613 }
9614 let outcome = match mode {
9615 MergeMode::None => MergeOutcome {
9616 mode,
9617 ok: true,
9618 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9619 empty: false,
9620 },
9621 MergeMode::Pr | MergeMode::Local
9622 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9623 {
9624 MergeOutcome {
9625 mode,
9626 ok: false,
9627 detail: empty_candidate_detail(&self.state, &base),
9628 empty: true,
9629 }
9630 }
9631 MergeMode::Local => {
9632 let on = git::current_branch(&repo).await?;
9633 if on.as_deref() != Some(base.as_str()) {
9634 MergeOutcome {
9635 mode,
9636 ok: false,
9637 detail: format!(
9638 "{} has {} checked out, not the base branch {base}",
9639 repo.display(),
9640 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9641 ),
9642 empty: false,
9643 }
9644 } else if !git::is_clean(&repo).await? {
9645 MergeOutcome {
9646 mode,
9647 ok: false,
9648 detail: format!("{} is dirty; refusing to merge", repo.display()),
9649 empty: false,
9650 }
9651 } else {
9652 let out = match style {
9653 MergeStyle::Merge => {
9654 git::merge_no_ff(&repo, &winner.branch, &message).await?
9655 }
9656 MergeStyle::Squash => {
9657 git::merge_squash(&repo, &winner.branch, &message).await?
9658 }
9659 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9660 };
9661 MergeOutcome {
9662 mode,
9663 ok: out.ok(),
9664 detail: if out.ok() { out.stdout } else { out.stderr },
9665 empty: false,
9666 }
9667 }
9668 }
9669 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9670 self.state.pr_publication_pending = true;
9671 self.state.save()?;
9672 let out =
9673 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9674 self.state.pr_publication_pending = out.is_err();
9675 MergeOutcome {
9676 mode,
9677 ok: out.is_ok(),
9678 detail: match out {
9679 Ok(url) => url,
9680 Err(e) => format!("{e:#}"),
9681 },
9682 empty: false,
9683 }
9684 }
9685 MergeMode::Pr => {
9686 let remote = self.state.config.merge.remote.clone();
9687 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9688 let pushed = if self.state.foreign_checkout.is_some() {
9689 let refspec = format!("{delivery_head}:refs/heads/{public}");
9690 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9691 } else {
9692 git::push(&winner.worktree, &remote, &winner.branch).await?
9693 };
9694 if !pushed.ok() {
9695 MergeOutcome {
9696 mode,
9697 ok: false,
9698 detail: pushed.stderr,
9699 empty: false,
9700 }
9701 } else {
9702 if self.state.foreign_checkout.is_some() {
9703 self.state.pr_pushed_head = Some(delivery_head.clone());
9704 }
9705 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9711 let out = match pr_merge_plan(found) {
9712 PrPlan::Create => {
9713 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9714 .await
9715 }
9716 PrPlan::Adopt { url, title } => {
9717 self.state
9718 .event("merge", format!("Pr: adopted open pull request {url}"));
9719 if self.state.explicit_github_text_selection() {
9720 land::set_pr_text_verbatim(
9723 &winner.worktree,
9724 &url,
9725 &pr.title,
9726 &pr.body,
9727 )
9728 .await
9729 .map(|()| url)
9730 } else {
9731 if title != pr.title
9732 && (!is_review_run(&self.state)
9733 || leaked
9734 .as_deref()
9735 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9736 && let Err(e) =
9737 if self.state.github_text.as_ref().is_some_and(|g| {
9738 g.resolved
9739 && g.title
9740 && g.choice == Some(GithubTextChoice::Original)
9741 }) {
9742 land::set_pr_title_verbatim(
9744 &winner.worktree,
9745 &url,
9746 &pr.title,
9747 )
9748 .await
9749 } else {
9750 land::set_pr_title(
9751 &mut self.state,
9752 &winner.worktree,
9753 &url,
9754 &pr.title,
9755 )
9756 .await
9757 }
9758 {
9759 tracing::warn!("could not refresh title of {url}: {e:#}");
9760 self.state
9761 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9762 }
9763 Ok(url)
9764 }
9765 }
9766 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9767 };
9768 match out {
9769 Ok(url) => MergeOutcome {
9770 mode,
9771 ok: true,
9772 detail: url,
9773 empty: false,
9774 },
9775 Err(e) => MergeOutcome {
9776 mode,
9777 ok: false,
9778 detail: e.to_string(),
9779 empty: false,
9780 },
9781 }
9782 }
9783 }
9784 };
9785
9786 self.state.status = match (mode, outcome.ok) {
9787 (MergeMode::None, _) => RunStatus::Ready,
9788 (_, true) if outcome.empty => RunStatus::Ready,
9789 (MergeMode::Pr, true) => RunStatus::Ready,
9790 (_, true) => RunStatus::Merged,
9791 (_, false) => RunStatus::Blocked,
9792 };
9793 self.state.event(
9794 "merge",
9795 format!(
9796 "{:?}: {}",
9797 mode,
9798 outcome.detail.lines().next().unwrap_or("")
9799 ),
9800 );
9801 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9802 self.state.record_github_text_publication(&outcome.detail);
9803 }
9804 self.state.merge = Some(outcome);
9805 self.state.save()?;
9806
9807 if self.state.config.graph.land
9813 && mode == MergeMode::Pr
9814 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9815 {
9816 self.run_land().await?;
9817 }
9818 self.settle_questions();
9823 Ok(())
9824 }
9825
9826 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9837 Box::pin(self.run_land_inner())
9838 }
9839
9840 async fn run_land_inner(&mut self) -> Result<()> {
9841 crate::pr_head::ensure_owner(&self.state)?;
9842 let url = self
9843 .state
9844 .pr
9845 .as_ref()
9846 .map(|pr| pr.url.clone())
9847 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9848 .unwrap_or_default();
9849 let url = url.lines().next().unwrap_or("").trim().to_owned();
9850 if !url.starts_with("http") {
9851 return Ok(());
9852 }
9853 loop {
9856 match land::land(&mut self.state, &url).await {
9857 Ok(_)
9858 if self
9859 .state
9860 .merge_repairs
9861 .last()
9862 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9863 {
9864 if self.repair_merge_review().await? {
9865 continue;
9866 }
9867 }
9868 Ok(_) if self.state.resource_block.is_some() => {
9869 return self.stop_for_resource();
9870 }
9871 Ok(pr) if self.state.parked => {
9872 let _ = pr;
9876 }
9877 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9878 self.state.status = RunStatus::Superseded;
9879 self.state.save()?;
9880 }
9881 Ok(pr) => {
9882 self.state.status = match pr.state {
9883 land::PrLifecycle::Merged => RunStatus::Merged,
9884 _ => RunStatus::Blocked,
9885 };
9886 if bump::should_release_bump(self.state.status)
9893 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9894 {
9895 self.state
9901 .event("bump", format!("release bump skipped: {e:#}"));
9902 }
9903 if self.state.status == RunStatus::Merged {
9906 crate::followup::after_merge(&mut self.state, &pr.url).await;
9907 }
9908 self.state.save()?;
9909 }
9910 Err(e) => {
9911 if e.is::<crate::repo_lock::ResourceBlocked>() {
9912 return Err(e);
9913 }
9914 self.state.status = RunStatus::Blocked;
9915 self.state.event("land", format!("gave up: {e}"));
9916 self.state.save()?;
9917 }
9918 }
9919 break;
9920 }
9921 Ok(())
9922 }
9923
9924 async fn repair_merge_review(&mut self) -> Result<bool> {
9927 use crate::run::MergeRepairPhase as Phase;
9928 crate::pr_head::ensure_owner(&self.state)?;
9929 let i = self.state.merge_repairs.len() - 1;
9930 let request = self.state.merge_repairs[i].clone();
9931 if request.phase == Phase::Fixing {
9932 let restored = match self.state.winner().cloned() {
9933 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9934 None => Err(anyhow::anyhow!("repair has no winning branch")),
9935 };
9936 let why = match restored {
9937 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9938 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9939 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9940 };
9941 self.hold_merge_repair(i, why)?;
9942 return Ok(false);
9943 }
9944 if request.phase == Phase::Reviewing && self.state.gate_ran {
9949 let winner = self
9950 .state
9951 .winner()
9952 .context("repair has no winning branch")?;
9953 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9954 if self.state.gate_head.as_deref() != Some(head.as_str())
9955 || !git::is_clean(&winner.worktree).await?
9956 {
9957 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9958 return Ok(false);
9959 }
9960 }
9961 let failure = if i >= self.state.config.graph.land_rounds
9962 || self.state.config.graph.review_rounds == 0
9963 {
9964 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9965 } else {
9966 None
9967 };
9968 if let Some(why) = failure {
9969 self.hold_merge_repair(i, why)?;
9970 return Ok(false);
9971 }
9972 if request.phase == Phase::Requested {
9973 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9974 Ok(pr) => pr,
9975 Err(e) => {
9976 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9977 return Ok(false);
9978 }
9979 };
9980 self.state.merge_repairs[i].phase = Phase::Fixing;
9981 self.state.land_approval = None;
9982 self.state.land_armed_head = None;
9983 self.state.save()?;
9984 let evidence = serde_json::to_string_pretty(&request)?;
9987 let reason = format!(
9988 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9989 );
9990 let budget = self.state.config.graph.land_rounds;
9991 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9992 match result {
9993 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9994 Ok(land::Fixed::Deferred) => {
9995 let worktree = self
9999 .state
10000 .winner()
10001 .context("repair has no winning branch")?
10002 .worktree
10003 .clone();
10004 self.graft_restore(&worktree, &request.head).await?;
10005 self.state.merge_repairs[i].phase = Phase::Requested;
10006 self.stop_for_resource()?;
10007 return Ok(false);
10008 }
10009 Ok(land::Fixed::Failed(why)) => {
10010 self.hold_merge_repair(
10011 i,
10012 format!(
10013 "repair failed: {why}; inspect the branch and run an explicit review"
10014 ),
10015 )?;
10016 return Ok(false);
10017 }
10018 Err(e) => {
10019 self.hold_merge_repair(
10020 i,
10021 format!(
10022 "repair failed: {e:#}; inspect the branch and run an explicit review"
10023 ),
10024 )?;
10025 return Ok(false);
10026 }
10027 }
10028 self.begin_merge_repair_review(i)?;
10032 }
10033 self.review_loop().await?;
10034 self.gate().await?;
10035 if self.resource_halt()? || self.state.parked {
10036 return Ok(false);
10037 }
10038 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10039 != Some(RunStatus::Gating)
10040 || !self.state.gate_allows_progress()
10041 {
10042 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10043 return Ok(false);
10044 }
10045 let winner = self
10048 .state
10049 .winner()
10050 .cloned()
10051 .context("repair has no winning branch")?;
10052 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10053 if self.state.gate_head.as_deref() != Some(after.as_str())
10054 || !git::is_clean(&winner.worktree).await?
10055 {
10056 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10057 return Ok(false);
10058 }
10059 let expected = self
10060 .state
10061 .pr_pushed_head
10062 .as_deref()
10063 .unwrap_or(&request.head)
10064 .to_owned();
10065 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10066 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10069 return Ok(false);
10070 }
10071 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10072 let refspec = format!("{after}:refs/heads/{public}");
10073 let push = git::git_raw(
10074 &winner.worktree,
10075 &["push", &lease, &self.state.config.merge.remote, &refspec],
10076 )
10077 .await;
10078 match push {
10079 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10080 Ok(out) => {
10081 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10082 return Ok(false);
10083 }
10084 Err(e) => {
10085 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10086 return Ok(false);
10087 }
10088 }
10089 self.state.merge_repairs[i].phase = Phase::Complete;
10090 self.state.status = RunStatus::Landing;
10091 self.state.event(
10092 "repair",
10093 "new review and gate completed; fresh owner merge decision required",
10094 );
10095 self.state.save()?;
10096 Ok(true)
10097 }
10098
10099 fn merge_repair_artifacts(&self) -> PathBuf {
10100 let artifacts = agent::artifacts_dir(&self.state.dir());
10101 match self
10102 .state
10103 .merge_repairs
10104 .last()
10105 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10106 {
10107 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10108 None => artifacts,
10109 }
10110 }
10111
10112 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10113 if let Some(e2e) = self.state.head_e2e.take() {
10114 self.state
10115 .verification_history
10116 .push(crate::run::VerificationEvidence {
10117 superseded_by: None,
10118 archived_at: Some(Timestamp::now()),
10119 fixes_spent: Some(self.state.gate_fixes.len()),
10120 gate_head: None,
10121 gate_ran: false,
10122 gate: Vec::new(),
10123 e2e: Some(e2e),
10124 });
10125 }
10126 self.state.reviews.clear();
10127 self.state.gate.clear();
10128 self.state.gate_ran = false;
10129 self.state.gate_head = None;
10130 self.state.gate_fixes.clear();
10131 self.state.gate_repair_e2e = None;
10132 self.state.pre_gate.clear();
10133 self.state.pre_gate_commit = None;
10134 self.state.contested_handoff = None;
10135 self.state.land_reruns = None;
10136 self.state.land_approval = None;
10137 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10138 self.state.status = RunStatus::Reviewing;
10139 self.state.save()
10140 }
10141
10142 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10143 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10144 self.state.merge_repairs[index].failure = Some(why.clone());
10145 self.state.status = RunStatus::Blocked;
10146 self.state.event("repair", why);
10147 self.state.save()
10148 }
10149
10150 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10154 if let Some(existing) = self.state.seats.get(key)
10155 && existing.agent == agent
10156 {
10157 return existing.clone();
10158 }
10159 let fresh = if self.state.seats.contains_key(key) {
10162 handover_seat(key, agent, self.state.next_seat_seed())
10163 } else {
10164 SeatState::new(key, agent, self.state.seed)
10165 };
10166 self.state.seats.insert(key.to_owned(), fresh.clone());
10167 fresh
10168 }
10169
10170 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10176 match self.state.seats.get(key) {
10177 Some(s) if s.agent != spec.id => {
10178 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10179 }
10180 _ => spec,
10181 }
10182 }
10183
10184 fn view(&self, c: &Candidate) -> CandidateView {
10186 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10187 .unwrap_or_default();
10188 let (patch, _) = blind::sanitize_patch(
10189 &format!("candidate {} patch", c.label),
10190 &raw,
10191 &self.state.config.blind,
10192 );
10193 CandidateView {
10194 label: c.label,
10195 branch: c.branch.clone(),
10196 summary: c.summary.clone(),
10197 stat: c.stat.clone(),
10198 patch,
10199 }
10200 }
10201
10202 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10204 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10205 prompt::judge(
10206 "(see above)",
10207 &views,
10208 self.roles.judges.len(),
10209 base_short,
10210 "en",
10211 )
10212 }
10213
10214 fn vote_prompt_full(
10220 &self,
10221 j: usize,
10222 labels: &[char],
10223 language: &str,
10224 candidates: &[Candidate],
10225 base_short: &str,
10226 ) -> String {
10227 let mut text = format!(
10228 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10229 prompt::final_vote(labels, language),
10230 self.state.instruction,
10231 self.candidate_block(candidates, base_short)
10232 );
10233 if let Some(own) = self
10234 .state
10235 .judgements
10236 .get(j)
10237 .filter(|r| !r.ranking.is_empty())
10238 {
10239 let reasons = own
10240 .reasons
10241 .iter()
10242 .map(|(k, v)| format!("- {k}: {v}"))
10243 .collect::<Vec<_>>()
10244 .join("\n");
10245 text.push_str(&format!(
10246 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10247 own.ranking.iter().collect::<String>(),
10248 if reasons.is_empty() {
10249 ""
10250 } else {
10251 ", because:\n"
10252 }
10253 ));
10254 }
10255 if !self.state.deliberation.is_empty() {
10256 text.push_str("\n# What was argued before this vote\n");
10257 for t in self.transcript(&[], j) {
10258 text.push_str(&format!(
10259 "\n## {}{}\n\n{}\n",
10260 t.who,
10261 if t.is_self { " (you)" } else { "" },
10262 t.body.trim()
10263 ));
10264 }
10265 }
10266 text
10267 }
10268
10269 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10276 let mut turns = Vec::new();
10277 for j in &self.state.judgements {
10278 if j.ranking.is_empty() {
10279 continue;
10280 }
10281 let reasons = j
10282 .reasons
10283 .iter()
10284 .map(|(k, v)| format!("- {k}: {v}"))
10285 .collect::<Vec<_>>()
10286 .join("\n");
10287 turns.push(Turn {
10288 who: format!("Judge {} (opening ranking)", j.judge),
10289 is_self: j.judge == self_idx + 1,
10290 body: format!(
10291 "Ranked {}{}{reasons}",
10292 j.ranking.iter().collect::<String>(),
10293 if reasons.is_empty() {
10294 ""
10295 } else {
10296 ", because:\n"
10297 }
10298 ),
10299 });
10300 }
10301 for t in self
10302 .state
10303 .deliberation
10304 .iter()
10305 .flat_map(|r| r.turns.iter())
10306 .chain(current)
10307 {
10308 turns.push(Turn {
10309 who: format!("Judge {}", t.judge),
10310 is_self: t.judge == self_idx + 1,
10311 body: t.body.clone(),
10312 });
10313 }
10314 turns
10315 }
10316}
10317
10318fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10320 agent::has_session(spec.kind, seat, sessions)
10321}
10322
10323fn next_untried_in_roster<'a>(
10344 roster: &'a [AgentSpec],
10345 start: usize,
10346 tried: &BTreeSet<String>,
10347) -> Option<&'a AgentSpec> {
10348 roster
10349 .get(start + 1..)?
10350 .iter()
10351 .find(|s| !tried.contains(&s.id))
10352}
10353
10354fn pick_successor<'a>(
10375 roster: &'a [AgentSpec],
10376 start: usize,
10377 tried: &BTreeSet<String>,
10378 carried: Option<&BTreeSet<String>>,
10379 others: &BTreeSet<String>,
10380) -> Option<&'a AgentSpec> {
10381 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10382 roster
10383 .get(start + 1..)
10384 .and_then(|tail| {
10385 tail.iter()
10386 .filter(free)
10387 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10388 })
10389 .or_else(|| {
10390 carried
10391 .is_some()
10392 .then(|| roster.iter().find(free))
10393 .flatten()
10394 })
10395 .or_else(|| match carried {
10396 Some(c) => next_for_seat(roster, start, tried, c),
10397 None => next_untried_in_roster(roster, start, tried),
10398 })
10399}
10400
10401fn next_for_seat<'a>(
10411 roster: &'a [AgentSpec],
10412 start: usize,
10413 round_tried: &BTreeSet<String>,
10414 carried_failed: &BTreeSet<String>,
10415) -> Option<&'a AgentSpec> {
10416 roster
10417 .get(start + 1..)?
10418 .iter()
10419 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10420 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10421}
10422
10423fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10429 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10430 return spec;
10431 };
10432 let ok = |id: &str| !h.failed.contains(id);
10433 if let Some(last) = h.last_ok.as_deref()
10434 && ok(last)
10435 && let Some(s) = roster.iter().find(|s| s.id == last)
10436 {
10437 return s.clone();
10438 }
10439 if ok(&spec.id) {
10440 return spec;
10441 }
10442 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10443 roster
10444 .get(from + 1..)
10445 .into_iter()
10446 .flatten()
10447 .chain(roster.iter())
10448 .find(|s| ok(&s.id))
10449 .cloned()
10450 .unwrap_or(spec)
10451}
10452
10453pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10457 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10458}
10459
10460const TIMED_OUT: &str = "timed out";
10463
10464impl FailClass {
10465 fn of(out: &AgentOutcome) -> Option<Self> {
10467 match out {
10468 AgentOutcome::Ok(_) => None,
10469 AgentOutcome::Quota(_) => Some(Self::Quota),
10470 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10471 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10472 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10473 AgentOutcome::ResourceBlocked(_) => None,
10476 }
10477 }
10478
10479 fn stem_word(&self) -> &'static str {
10481 match self {
10482 Self::Quota => "quota",
10483 _ => "handover",
10484 }
10485 }
10486}
10487
10488fn failure_signature(msg: &str) -> String {
10492 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10493 if let Some(code) = line
10496 .strip_prefix("process exited with no usable output (exit code ")
10497 .and_then(|s| s.strip_suffix(')'))
10498 .and_then(|s| s.parse::<i32>().ok())
10499 {
10500 line = format!("exited with some({code}) and no usable output");
10501 } else if matches!(
10502 line.as_str(),
10503 "process exited with no usable output (no exit code reported)"
10504 | "process exited with no usable output (exit code unavailable)"
10505 ) {
10506 line = "exited with none and no usable output".to_owned();
10507 }
10508 let mut out = Vec::new();
10509 for word in line.split_whitespace() {
10510 if word.contains('/') || word.contains('\\') {
10511 out.push("<path>".to_owned());
10512 continue;
10513 }
10514 let mut w = String::new();
10515 let mut in_digits = false;
10516 for c in word.chars() {
10517 if c.is_ascii_digit() {
10518 if !in_digits {
10519 w.push('#');
10520 }
10521 in_digits = true;
10522 } else {
10523 in_digits = false;
10524 w.push(c);
10525 }
10526 }
10527 out.push(w);
10528 }
10529 out.join(" ").chars().take(120).collect()
10530}
10531
10532fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10539 match cur {
10540 FailClass::Quota | FailClass::Timeout => true,
10541 FailClass::Other(_) => prev != Some(cur),
10542 }
10543}
10544
10545fn fail_reason(out: &AgentOutcome) -> String {
10547 match out {
10548 AgentOutcome::Ok(_) => String::new(),
10549 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10550 AgentOutcome::Dropped(o) => format!(
10551 "the CLI dropped the stream ({})",
10552 o.dropped
10553 .as_ref()
10554 .map(|d| d.why.as_str())
10555 .unwrap_or("it ended without delivering its answer")
10556 ),
10557 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10558 e.lines().next().unwrap_or("").chars().take(160).collect()
10559 }
10560 }
10561}
10562
10563pub(crate) fn record_handover(
10566 state: &mut RunState,
10567 node: &str,
10568 seat: &str,
10569 from: &str,
10570 to: &str,
10571 class: &FailClass,
10572 reason: &str,
10573) {
10574 let message = if *class == FailClass::Quota {
10575 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10576 } else {
10577 format!("{seat}: handed over {from} -> {to} ({reason})")
10578 };
10579 state.event(node, message);
10580 state.handovers.push(Handover {
10581 at: Timestamp::now(),
10582 node: node.to_owned(),
10583 seat: seat.to_owned(),
10584 from: from.to_owned(),
10585 to: to.to_owned(),
10586 reason: reason.to_owned(),
10587 });
10588}
10589
10590fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10604 commands.iter().any(|c| c.exit_code.is_none())
10605}
10606
10607fn verified_noop_claim(
10620 usable: bool,
10621 commands: &[agent::CommandEvidence],
10622 text: &str,
10623) -> Option<String> {
10624 (usable && !has_unconfirmed_command(commands))
10625 .then(|| verdict::verified_noop(text))
10626 .flatten()
10627}
10628
10629fn short(commit: &str) -> String {
10630 commit.chars().take(7).collect()
10631}
10632
10633fn make_executable(path: &Path) -> Result<()> {
10634 #[cfg(unix)]
10635 {
10636 use std::os::unix::fs::PermissionsExt as _;
10637 let mut perms = std::fs::metadata(path)?.permissions();
10638 perms.set_mode(0o755);
10639 std::fs::set_permissions(path, perms)?;
10640 }
10641 #[cfg(not(unix))]
10642 {
10643 let _ = path;
10644 }
10645 Ok(())
10646}
10647
10648struct WaveCtx<'a> {
10655 run: &'a str,
10658 node: &'a str,
10660 prompts: &'a Prompts,
10661 cache: Option<&'a Path>,
10663 round: Option<usize>,
10666 carry_seats: bool,
10670 pauses: Option<(&'a Pause, &'a Pause)>,
10673}
10674
10675fn repository_wait(
10677 state: &RunState,
10678 pauses: Option<(&Pause, &Pause)>,
10679) -> crate::repo_lock::WaitPolicy {
10680 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10681 let inherited = crate::repo_lock::WaitPolicy::current();
10682 crate::repo_lock::WaitPolicy::new(
10683 Duration::from_secs(state.config.graph.timeout_resource_wait),
10684 move || {
10685 inherited.cancelled()
10686 || pauses
10687 .as_ref()
10688 .is_some_and(|(a, b)| a.parked() || b.parked())
10689 },
10690 )
10691}
10692
10693fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10694 if error.is::<crate::repo_lock::ResourceBlocked>() {
10695 state.resource_block = Some(crate::run::ResourceBlock {
10696 cause: crate::run::ResourceBlockCause::Other,
10697 node: "repository".into(),
10698 kind: if error.to_string().contains("disk pressure") {
10699 "disk pressure"
10700 } else if error.to_string().contains("disk admission") {
10701 "disk admission"
10702 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10703 "incremental cache admission"
10704 } else {
10705 "repository coordination"
10706 }
10707 .into(),
10708 detail: error.to_string(),
10709 at: Timestamp::now(),
10710 });
10711 state.event("resource", error.to_string());
10712 }
10713}
10714
10715async fn coordinate_operation(
10716 state: &mut RunState,
10717 cwd: &Path,
10718 wait: &crate::repo_lock::WaitPolicy,
10719) -> Result<Option<crate::repo_lock::SeatLease>> {
10720 let result =
10721 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10722 .await;
10723 if let Err(e) = &result {
10724 record_repository_block(state, e);
10725 }
10726 result
10727}
10728
10729async fn run_one(
10731 job: SeatJob,
10732 sem: Arc<Semaphore>,
10733 ctx: &WaveCtx<'_>,
10734 state: &mut RunState,
10735 attempt: usize,
10736) -> (SeatState, AgentOutcome) {
10737 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10738 .await
10739 .pop()
10740 .expect("one job in, one result out");
10741 (seat, out)
10742}
10743
10744async fn wave(
10750 jobs: Vec<SeatJob>,
10751 sem: Arc<Semaphore>,
10752 ctx: &WaveCtx<'_>,
10753 state: &mut RunState,
10754 attempt: usize,
10755) -> Vec<(usize, SeatState, AgentOutcome)> {
10756 let WaveCtx {
10757 run,
10758 node,
10759 prompts,
10760 cache,
10761 round,
10762 carry_seats: _,
10763 pauses,
10764 } = *ctx;
10765 for job in &jobs {
10766 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10767 }
10768 if let Err(e) = state.save() {
10769 tracing::warn!("could not persist in-progress seats: {e:#}");
10774 }
10775 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10791 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10795 let cache_guard = if let Some(cache_dir) = cache {
10796 if jobs_had_a_writer {
10797 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10798 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10799 match acquire_cache_lease(
10800 state,
10801 cache_dir,
10802 &owner,
10803 resource_limit,
10804 node,
10805 node,
10806 &cancel,
10807 )
10808 .await
10809 {
10810 Ok(Some(g)) => Some(g),
10811 Ok(None) => {
10814 if let Some(block) = &mut state.resource_block {
10815 block.cause = crate::run::ResourceBlockCause::Cancelled {
10816 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10817 };
10818 }
10819 return block_wave(
10820 jobs,
10821 state,
10822 node,
10823 attempt,
10824 "parked while waiting for the shared build cache",
10825 );
10826 }
10827 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10830 Err(e) => {
10831 let why = format!("{e:#}");
10835 state.event(
10836 "resource",
10837 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10838 );
10839 return block_wave(jobs, state, node, attempt, &why);
10840 }
10841 }
10842 } else {
10843 None
10844 }
10845 } else {
10846 None
10847 };
10848 state.seats_spawning(node, attempt);
10851 if let Err(e) = state.save() {
10852 tracing::warn!("could not persist the start of a wave: {e:#}");
10853 }
10854 let slot_limit = resource_limit
10858 + jobs
10859 .iter()
10860 .map(|j| j.timeout)
10861 .max()
10862 .unwrap_or_default()
10863 .saturating_mul(jobs.len() as u32);
10864 let mut set = tokio::task::JoinSet::new();
10865 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10868 let overlay = prompts.overlay(node);
10869 let repository_wait = repository_wait(state, pauses);
10870 for (i, mut job) in jobs.into_iter().enumerate() {
10871 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10872 if cache.is_some() {
10873 job.prompt.push('\n');
10874 job.prompt
10875 .push_str(&prompt::build_cache_note(node, job.allow_write));
10876 }
10877 let sem = Arc::clone(&sem);
10878 let run = run.to_owned();
10879 let node = node.to_owned();
10880 let attachments = if node == "implement" {
10883 state.attachments.clone()
10884 } else {
10885 Vec::new()
10886 };
10887 let cache = cache
10898 .filter(|_| job.allow_write && cache_guard.is_some())
10899 .map(Path::to_path_buf);
10900 let spawned_tx = spawned_tx.clone();
10901 let repository_wait = repository_wait.clone();
10902 let coordination = state.config.coordination.clone();
10903 let fetched_base = state.fetched_base.clone();
10904 set.spawn(async move {
10905 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10909 Ok(p) => p,
10910 Err(_) => {
10911 return (
10912 i,
10913 job.seat,
10914 AgentOutcome::ResourceBlocked(format!(
10915 "no parallel slot became free within {}s",
10916 slot_limit.as_secs()
10917 )),
10918 false,
10919 );
10920 }
10921 };
10922 spawned_tx.send(job.seat.key.clone()).ok();
10923 let mut seat = job.seat;
10924 let out = repository_wait
10925 .scope(agent::invoke(
10926 &job.spec,
10927 &mut seat,
10928 &Invocation {
10929 coordination: coordination.as_ref(),
10930 fetched_base: fetched_base.as_ref(),
10931 cwd: &job.cwd,
10932 prompt: &job.prompt,
10933 timeout: job.timeout,
10934 allow_write: job.allow_write,
10935 sessions: job.sessions,
10936 artifacts: &job.artifacts,
10937 stem: &job.stem,
10938 run: &run,
10939 node: &node,
10940 cache_dir: cache.as_deref(),
10941 attachments: &attachments,
10942 writable: &[],
10943 },
10944 ))
10945 .await;
10946 let interrupted = out
10947 .as_ref()
10948 .err()
10949 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10950 let out = match out {
10951 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10952 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10953 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10961 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10962 Ok(o) => AgentOutcome::Failed(format!(
10963 "process exited with no usable output ({})",
10964 crate::notices::exit_status(o.exit_code)
10965 )),
10966 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10967 AgentOutcome::ResourceBlocked(e.to_string())
10968 }
10969 Err(e) => AgentOutcome::Failed(e.to_string()),
10970 };
10971 (i, seat, out, interrupted)
10972 });
10973 }
10974 drop(spawned_tx);
10975 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10976 loop {
10977 let joined = tokio::select! {
10978 Some(key) = spawned_rx.recv() => {
10979 state.seat_spawned(&key);
10981 if let Err(e) = state.save() {
10982 tracing::warn!("could not persist a seat's start: {e:#}");
10983 }
10984 continue;
10985 }
10986 joined = set.join_next() => joined,
10987 };
10988 let Some(joined) = joined else {
10989 break;
10990 };
10991 let (i, seat, out, interrupted) = match joined {
10992 Ok(v) => v,
10993 Err(e) => {
10997 tracing::error!("agent task panicked: {e}");
10998 continue;
10999 }
11000 };
11001 state.seat_finished(&seat.key);
11002 if interrupted
11003 && node == "implement"
11004 && let Some(label) = state
11005 .candidates
11006 .iter()
11007 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11008 .map(|candidate| candidate.label)
11009 {
11010 state.candidate_preservation.insert(
11011 label,
11012 "interrupted implementation awaiting persistence".into(),
11013 );
11014 state.interrupted_implementations.insert(label);
11015 }
11016 if let AgentOutcome::ResourceBlocked(why) = &out
11017 && state.resource_block.is_none()
11018 {
11019 state.resource_block = Some(crate::run::ResourceBlock {
11020 cause: crate::run::ResourceBlockCause::Other,
11021 node: node.to_owned(),
11022 kind: if why.contains("disk pressure") {
11023 "disk pressure"
11024 } else if why.contains("disk admission") {
11025 "disk admission"
11026 } else if why.contains(crate::incremental::BLOCKED) {
11027 "incremental cache admission"
11028 } else if why.contains("repository coordination") {
11029 "repository coordination"
11030 } else {
11031 "parallel slot"
11032 }
11033 .to_owned(),
11034 detail: why.clone(),
11035 at: Timestamp::now(),
11036 });
11037 let activity = if interrupted {
11038 "agent execution interrupted"
11039 } else {
11040 "no agent was started"
11041 };
11042 state.event("resource", format!("{node}: {activity} - {why}"));
11043 }
11044 record_jobs(state, node, round, &seat.key, &out);
11045 if let Err(e) = state.save() {
11046 tracing::warn!("could not persist a seat's completion: {e:#}");
11047 }
11048 if collected.len() <= i {
11049 collected.resize_with(i + 1, || None);
11050 }
11051 collected[i] = Some((i, seat, out));
11052 }
11053 if state
11059 .active
11060 .values()
11061 .any(|a| a.node == node && a.attempt == attempt)
11062 {
11063 state
11064 .active
11065 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11066 if let Err(e) = state.save() {
11067 tracing::warn!("could not persist the end of a wave: {e:#}");
11068 }
11069 }
11070 if let Some(cache_dir) = cache
11077 && jobs_had_a_writer
11078 {
11079 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11080 }
11081 if let Some(guard) = cache_guard {
11082 guard.release();
11083 }
11084 collected.into_iter().flatten().collect()
11085}
11086
11087fn record_jobs(
11098 state: &mut RunState,
11099 node: &str,
11100 round: Option<usize>,
11101 seat: &str,
11102 out: &AgentOutcome,
11103) {
11104 let commands: &[agent::CommandEvidence] = match out {
11105 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11106 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11107 };
11108 let checked_at = Timestamp::now();
11109 for c in commands {
11110 state.jobs.push(JobRecord {
11111 node: node.to_owned(),
11112 round,
11113 seat: seat.to_owned(),
11114 id: c.id.clone(),
11115 description: c.description.clone(),
11116 checked_at,
11117 status: match c.exit_code {
11118 Some(0) => JobStatus::Completed,
11119 Some(_) => JobStatus::Failed,
11120 None => JobStatus::Unknown,
11121 },
11122 exit_code: c.exit_code,
11123 result_summary: c.result_summary.clone(),
11124 source: c.source.clone(),
11125 });
11126 }
11127}
11128
11129fn round_is_clean(
11150 blocking: usize,
11151 e2e_ok: bool,
11152 answered: usize,
11153 expected: usize,
11154 quota_missing: usize,
11155 policy: IncompleteReviewPolicy,
11156) -> bool {
11157 if blocking != 0 || !e2e_ok {
11158 return false;
11159 }
11160 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11161 return true;
11162 }
11163 answered > 0 && expected - answered <= quota_missing
11164}
11165
11166pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11190 if max_rounds == 0
11191 || reviews.last().is_some_and(|r| {
11192 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11193 })
11194 {
11195 return Some(RunStatus::Gating);
11196 }
11197 let last = reviews.last()?;
11198 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11199 if reviews.len() < max_rounds && !stagnant {
11200 return None;
11201 }
11202 if last.incomplete() && last.blocking == 0 {
11203 return Some(RunStatus::Blocked);
11204 }
11205 if last.e2e_status() == E2eStatus::ResourceBlocked
11206 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11207 {
11208 return None;
11209 }
11210 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11211 RunStatus::Gating
11212 } else {
11213 RunStatus::Blocked
11214 })
11215}
11216
11217pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11232 if nudged {
11233 (full / 4).max(Duration::from_secs(120)).min(full)
11234 } else {
11235 full
11236 }
11237}
11238
11239#[allow(clippy::too_many_arguments)]
11268async fn ask_json_wave<T>(
11269 jobs: Vec<SeatJob>,
11270 sem: Arc<Semaphore>,
11271 retries: usize,
11272 roster: &[AgentSpec],
11273 ctx: &WaveCtx<'_>,
11274 losses: &mut Vec<QuotaLoss>,
11275 state: &mut RunState,
11276 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11277) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11278where
11279 T: serde::de::DeserializeOwned + Send + 'static,
11280{
11281 ask_wave_with(
11282 jobs,
11283 sem,
11284 retries,
11285 roster,
11286 ctx,
11287 losses,
11288 state,
11289 &|text: &str| {
11290 let v = verdict::extract_json::<T>(text)?;
11291 validate(&v)?;
11292 Ok(v)
11293 },
11294 )
11295 .await
11296}
11297
11298#[allow(clippy::too_many_arguments)]
11307async fn ask_wave_with<T>(
11308 jobs: Vec<SeatJob>,
11309 sem: Arc<Semaphore>,
11310 retries: usize,
11311 roster: &[AgentSpec],
11312 ctx: &WaveCtx<'_>,
11313 losses: &mut Vec<QuotaLoss>,
11314 state: &mut RunState,
11315 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11316) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11317where
11318 T: Send + 'static,
11319{
11320 let n = jobs.len();
11321 let originals: Vec<SeatJob> = jobs;
11322 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11323 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11324 let mut nudges: Vec<usize> = vec![0; n];
11331 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11335 let mut tried: Vec<BTreeSet<String>> = specs
11336 .iter()
11337 .map(|s| BTreeSet::from([s.id.clone()]))
11338 .collect();
11339 let starts: Vec<usize> = specs
11340 .iter()
11341 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11342 .collect();
11343 let carry = ctx.carry_seats && !roster.is_empty();
11344 let carried: Vec<BTreeSet<String>> = originals
11347 .iter()
11348 .map(|j| {
11349 state
11350 .seat_history
11351 .get(&j.seat.key)
11352 .filter(|_| carry)
11353 .map(|h| h.failed.clone())
11354 .unwrap_or_default()
11355 })
11356 .collect();
11357 let mut prev: Vec<Option<FailClass>> = originals
11358 .iter()
11359 .map(|j| {
11360 state
11361 .seat_history
11362 .get(&j.seat.key)
11363 .filter(|_| carry)
11364 .and_then(|h| h.last_fail.clone())
11365 })
11366 .collect();
11367 let next_agent =
11368 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11369 let others: BTreeSet<String> = specs
11370 .iter()
11371 .enumerate()
11372 .filter(|(j, _)| *j != i)
11373 .map(|(_, s)| s.id.clone())
11374 .collect();
11375 pick_successor(
11376 roster,
11377 starts[i],
11378 tried,
11379 carry.then(|| &carried[i]),
11380 &others,
11381 )
11382 .cloned()
11383 };
11384 let mut fresh: Vec<Option<String>> = vec![None; n];
11385 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11386 let mut pending: Vec<usize> = (0..n).collect();
11387
11388 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11393 for round in 0..max_rounds {
11394 if pending.is_empty() {
11395 break;
11396 }
11397 let mut batch = Vec::with_capacity(pending.len());
11398 let mut renudged: Vec<&str> = Vec::new();
11399 for &i in &pending {
11400 let src = &originals[i];
11401 let full: &str = match &src.handover {
11406 Some(h) if specs[i].id != src.spec.id => h,
11407 _ => &src.prompt,
11408 };
11409 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11412 (
11413 full.to_owned(),
11414 src.timeout,
11415 format!("{}-{word}-{}", src.stem, specs[i].id),
11416 )
11417 } else if nudges[i] == 0 {
11418 (full.to_owned(), src.timeout, src.stem.clone())
11419 } else {
11420 renudged.push(src.seat.key.as_str());
11421 let why = done[i]
11422 .as_ref()
11423 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11424 .unwrap_or_else(|| "no parsable answer".to_owned());
11425 let nudge = prompt::nudge(&why);
11426 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11427 let prompt = if nudged {
11428 nudge
11429 } else {
11430 format!("{full}\n\n---\n\n{nudge}")
11431 };
11432 (
11433 prompt,
11434 retry_budget(src.timeout, nudged),
11435 format!("{}-retry{}", src.stem, nudges[i]),
11436 )
11437 };
11438 batch.push(SeatJob {
11439 spec: specs[i].clone(),
11440 seat: seats[i].clone(),
11441 cwd: src.cwd.clone(),
11442 prompt,
11443 timeout,
11444 allow_write: src.allow_write,
11445 sessions: src.sessions,
11446 artifacts: src.artifacts.clone(),
11447 stem,
11448 handover: None,
11449 });
11450 }
11451
11452 if !renudged.is_empty() {
11453 state.event(
11454 ctx.node,
11455 format!("retry {round}: re-asking {}", renudged.join(", ")),
11456 );
11457 }
11458 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11459 let mut still = Vec::new();
11460 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11461 seats[i] = seat;
11462 if let AgentOutcome::ResourceBlocked(why) = &out {
11465 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11466 continue;
11467 }
11468 let class = FailClass::of(&out);
11469 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11472 && nudges[i] < retries
11473 && !roster.is_empty();
11474 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11475 let next = should_hand_over(prev[i].as_ref(), &cur)
11476 .then(|| next_agent(i, &tried[i], &specs))
11477 .flatten();
11478 if carry {
11479 let h = state
11480 .seat_history
11481 .entry(originals[i].seat.key.clone())
11482 .or_default();
11483 h.failed.insert(specs[i].id.clone());
11484 h.last_fail = Some(cur.clone());
11485 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11486 h.last_ok = None;
11487 }
11488 if let Err(e) = state.save() {
11491 tracing::warn!("could not persist a seat's failure history: {e:#}");
11492 }
11493 }
11494 if let Some(next) = next {
11495 record_handover(
11496 state,
11497 ctx.node,
11498 &originals[i].seat.key,
11499 &specs[i].id,
11500 &next.id,
11501 &cur,
11502 &fail_reason(&out),
11503 );
11504 tried[i].insert(next.id.clone());
11505 prev[i] = Some(cur.clone());
11506 seats[i] =
11507 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11508 specs[i] = next;
11509 fresh[i] = Some(cur.stem_word().to_owned());
11510 nudges[i] = 0;
11511 done[i] = Some(Err(anyhow::anyhow!(
11512 "handed over after: {}",
11513 fail_reason(&out)
11514 )));
11515 still.push(i);
11516 continue;
11517 }
11518 }
11519 if carry
11520 && !nudge_first
11521 && let Some(cur) = class.clone()
11522 {
11523 let h = state
11525 .seat_history
11526 .entry(originals[i].seat.key.clone())
11527 .or_default();
11528 h.failed.insert(specs[i].id.clone());
11529 h.last_fail = Some(cur);
11530 }
11531 let parsed = match out {
11532 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11533 AgentOutcome::Quota(o) => {
11534 last_quota[i] = o.quota.clone();
11535 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11536 }
11537 AgentOutcome::Dropped(o) => {
11541 let why = o
11542 .dropped
11543 .as_ref()
11544 .map(|d| d.why.as_str())
11545 .unwrap_or("the CLI ended the stream without delivering its answer");
11546 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11547 }
11548 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11549 Err(anyhow::anyhow!(e))
11550 }
11551 };
11552 let quota = class == Some(FailClass::Quota);
11553 let failed = parsed.is_err();
11554 if !failed && matches!(ctx.node, "judge" | "vote") {
11555 state
11558 .quota
11559 .retain(|loss| loss.seat != originals[i].seat.key);
11560 }
11561 done[i] = Some(parsed);
11562 if carry && !failed {
11563 let h = state
11564 .seat_history
11565 .entry(originals[i].seat.key.clone())
11566 .or_default();
11567 h.failed.remove(&specs[i].id);
11568 h.last_ok = Some(specs[i].id.clone());
11569 h.last_fail = None;
11570 }
11571 let agent_failure = class.is_some()
11579 && !nudge_first
11580 && !roster.is_empty()
11581 && next_agent(i, &tried[i], &specs).is_some();
11582 if failed && !quota && !agent_failure && nudges[i] < retries {
11583 nudges[i] += 1;
11584 still.push(i);
11585 }
11586 }
11587 pending = still;
11588 }
11589
11590 for (i, q) in last_quota.into_iter().enumerate() {
11593 if let Some(quota) = q {
11594 losses.push(QuotaLoss::from_report(
11595 originals[i].seat.key.clone(),
11596 ctx.node,
11597 Some("a),
11598 ));
11599 }
11600 }
11601
11602 if state.status != RunStatus::Stalled
11605 && ctx.node != "deliberate"
11606 && n > 0
11607 && losses.len() == n
11608 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11609 {
11610 block_machine_outage(state, ctx.node, losses);
11611 state.quota.append(losses);
11612 }
11613
11614 seats
11615 .into_iter()
11616 .zip(done)
11617 .zip(nudges)
11618 .map(|((seat, res), attempts)| {
11619 (
11620 seat,
11621 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11622 attempts,
11623 )
11624 })
11625 .collect()
11626}
11627
11628pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11631 let mut auth: Vec<_> = losses
11632 .iter()
11633 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11634 .map(|loss| loss.agent.clone())
11635 .collect();
11636 auth.sort();
11637 auth.dedup();
11638 let (kind, detail) = if auth.is_empty() {
11639 (
11640 "agent usage limit",
11641 "All seats reached their agent usage limits; resume after usage is available"
11642 .to_owned(),
11643 )
11644 } else {
11645 (
11646 "agent authentication",
11647 format!(
11648 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11649 auth.join(", ")
11650 ),
11651 )
11652 };
11653 state.resource_block = Some(crate::run::ResourceBlock {
11654 node: node.to_owned(),
11655 kind: kind.to_owned(),
11656 cause: crate::run::ResourceBlockCause::Other,
11657 detail: detail.clone(),
11658 at: Timestamp::now(),
11659 });
11660 state.event(node, detail);
11661}
11662
11663fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11665 results
11666 .iter()
11667 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11668}
11669
11670fn block_wave(
11674 jobs: Vec<SeatJob>,
11675 state: &mut RunState,
11676 node: &str,
11677 attempt: usize,
11678 why: &str,
11679) -> Vec<(usize, SeatState, AgentOutcome)> {
11680 state.resource_wait = None;
11681 state
11682 .active
11683 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11684 if let Err(e) = state.save() {
11685 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11686 }
11687 jobs.into_iter()
11688 .enumerate()
11689 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11690 .collect()
11691}
11692
11693async fn acquire_cache_lease(
11706 state: &mut RunState,
11707 cache_dir: &Path,
11708 owner: &crate::cache::Owner,
11709 budget: Duration,
11710 node: &str,
11711 context: &str,
11712 cancel: &(dyn Fn() -> bool + Sync),
11713) -> Result<Option<crate::cache::Guard>> {
11714 let home = crate::run::home();
11715 let started = Instant::now();
11716 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11717 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11718 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11719 Err(e) => {
11720 state.event(
11721 "verify",
11722 format!("{context}: could not check the shared build cache: {e:#}"),
11723 );
11724 if let Err(e2) = state.save() {
11725 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11726 }
11727 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11728 }
11729 };
11730 state.event(
11731 "verify",
11732 format!(
11733 "{context}: waiting for the shared build cache at {} ({})",
11734 cache_dir.display(),
11735 busy.describe()
11736 ),
11737 );
11738 state.resource_wait = Some(crate::run::ResourceWait {
11741 node: context.to_owned(),
11742 kind: "shared build cache".to_owned(),
11743 holder: busy.describe(),
11744 since: Timestamp::now(),
11745 limit_secs: budget.as_secs(),
11746 });
11747 if let Err(e) = state.save() {
11748 tracing::warn!("could not persist a cache wait: {e:#}");
11749 }
11750 let remaining = budget.saturating_sub(started.elapsed());
11751 let waited = crate::cache::wait_for_unless(
11752 &home,
11753 cache_dir,
11754 owner,
11755 remaining,
11756 Duration::from_secs(5),
11757 cancel,
11758 )
11759 .await;
11760 state.resource_wait = None;
11761 let cause = match &waited {
11762 Ok(Some(_)) => None,
11763 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11764 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11765 Some(crate::run::ResourceBlockCause::Timeout {
11766 waited_secs: started.elapsed().as_secs(),
11767 limit_secs: budget.as_secs(),
11768 })
11769 }
11770 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11771 };
11772 if let Some(cause) = cause {
11773 let detail = match &waited {
11774 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11775 Err(error) => format!("{error:#}"),
11776 };
11777 state.resource_block = Some(crate::run::ResourceBlock {
11778 cause,
11779 node: node.to_owned(),
11780 kind: "shared build cache".into(),
11781 detail,
11782 at: Timestamp::now(),
11783 });
11784 state.event(
11785 "resource",
11786 state.resource_block.as_ref().unwrap().description(),
11787 );
11788 }
11789 if let Err(error) = state.save() {
11790 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11791 }
11792 waited
11793}
11794
11795#[derive(Debug)]
11800struct CacheCheckError(String);
11801
11802impl std::fmt::Display for CacheCheckError {
11803 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11804 f.write_str(&self.0)
11805 }
11806}
11807
11808impl std::error::Error for CacheCheckError {}
11809
11810fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11812 state
11813 .resource_block
11814 .get_or_insert_with(|| crate::run::ResourceBlock {
11815 cause: crate::run::ResourceBlockCause::Other,
11816 node: node.to_owned(),
11817 kind: "shared build cache".into(),
11818 detail: format!("{error:#}"),
11819 at: Timestamp::now(),
11820 });
11821 if let Err(error) = state.save() {
11822 tracing::warn!("could not persist cache resource block: {error:#}");
11823 }
11824}
11825
11826#[allow(clippy::too_many_arguments)]
11847async fn with_cache_lease<'s, F, Fut>(
11848 state: &'s mut RunState,
11849 cache_dir: Option<&Path>,
11850 node: &str,
11851 seat: &str,
11852 worktree: &Path,
11853 head: &str,
11854 budget: Duration,
11855 context: &str,
11856 wait: &crate::repo_lock::WaitPolicy,
11857 body: F,
11858) -> Result<(Vec<CommandOutcome>, bool)>
11859where
11860 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11861 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11862{
11863 let started = Instant::now();
11866 let home = crate::run::home();
11867 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11868 let guard = if let Some(cache_dir) = cache_dir {
11869 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11870 match acquire_cache_lease(
11871 state,
11872 cache_dir,
11873 &owner,
11874 resource_limit,
11875 node,
11876 context,
11877 &|| wait.cancelled(),
11878 )
11879 .await
11880 .and_then(|g| {
11881 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11882 }) {
11883 Ok(g) => Some(g),
11884 Err(e) => {
11885 record_cache_error(state, node, &e);
11886 return Ok((
11887 vec![CommandOutcome {
11888 command: "(waiting for the shared build cache)".into(),
11889 code: None,
11890 output_tail: e.to_string(),
11891 duration_ms: started.elapsed().as_millis() as u64,
11892 resource_blocked: true,
11893 }],
11894 false,
11895 ));
11896 }
11897 }
11898 } else {
11899 None
11900 };
11901 let _coordination = coordinate_operation(state, worktree, wait).await?;
11902 if let Some(cache_dir) = cache_dir {
11903 let identity = crate::cache::Identity::new(worktree, head);
11904 if let Err(e) = crate::cache::ensure_fresh(
11905 &home,
11906 cache_dir,
11907 &identity,
11908 &state.config.verify.cargo_manifests(),
11909 ) {
11910 record_cache_error(state, node, &e);
11911 state.event(
11912 "verify",
11913 format!(
11914 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11915 worktree.display(),
11916 short(head)
11917 ),
11918 );
11919 if let Some(guard) = guard {
11920 guard.release();
11921 }
11922 return Ok((
11923 vec![CommandOutcome {
11924 command: "(confirming the shared build cache is fresh)".into(),
11925 code: None,
11926 output_tail: e.to_string(),
11927 duration_ms: started.elapsed().as_millis() as u64,
11928 resource_blocked: true,
11929 }],
11930 false,
11931 ));
11932 }
11933 }
11934 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11935 if !timed_out_pids.is_empty() {
11936 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11937 }
11938 drop(_coordination);
11941 if let Some(guard) = guard {
11942 guard.release();
11943 }
11944 Ok((outcomes, retried))
11945}
11946
11947async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11959 wait_for_pids_with(
11960 pids,
11961 crate::proc::pid_alive,
11962 LEASE_RELEASE_POLL,
11963 LEASE_RELEASE_MAX_WAIT,
11964 )
11965 .await;
11966}
11967
11968async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11974 pids: &[u32],
11975 alive: F,
11976 poll: Duration,
11977 max_wait: Duration,
11978) {
11979 let deadline = Instant::now() + max_wait;
11980 loop {
11981 if pids.iter().all(|&pid| !alive(pid)) {
11982 return;
11983 }
11984 if Instant::now() >= deadline {
11985 return;
11986 }
11987 tokio::time::sleep(poll).await;
11988 }
11989}
11990
11991fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11998 outcomes.iter().any(|o| o.resource_blocked)
11999}
12000
12001pub(crate) enum GateFix {
12003 Retry,
12005 Stop,
12008 Defer,
12011}
12012
12013pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12021 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12022 red.peek().is_some()
12023 && red.all(|o| {
12024 !o.resource_blocked
12025 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12026 && !o.output_tail.trim().is_empty()
12027 })
12028}
12029
12030fn e2e_outcome_label(o: &CommandOutcome) -> String {
12034 if o.ok() {
12035 return "pass".to_owned();
12036 }
12037 let reason = if o.build_failed() {
12038 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12039 } else {
12040 format!("FAIL ({:?})", o.code)
12041 };
12042 format!(
12043 "{reason}\n{}",
12044 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12045 )
12046}
12047
12048async fn run_e2e_with_retry(
12056 state: &mut RunState,
12057 shell: &[String],
12058 commands: &[String],
12059 worktree: &Path,
12060 timeout: Duration,
12061 context: &str,
12062) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12063 let (mut e2e, mut timed_out_pids) = run_commands(
12064 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12065 )
12066 .await;
12067 for o in &e2e {
12068 state.event(
12069 "verify",
12070 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12071 );
12072 }
12073 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12077 if verify_retried {
12078 state.event(
12079 "verify",
12080 format!(
12081 "{context}: verify could not build/link, not a test result — retrying once \
12082 before concluding"
12083 ),
12084 );
12085 let retried = run_commands(
12086 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12087 )
12088 .await;
12089 e2e = retried.0;
12090 timed_out_pids.extend(retried.1);
12093 for o in &e2e {
12094 state.event(
12095 "verify",
12096 format!(
12097 "{context}: retry `{}` -> {}",
12098 o.command,
12099 e2e_outcome_label(o)
12100 ),
12101 );
12102 }
12103 }
12104 (e2e, verify_retried, timed_out_pids)
12105}
12106
12107#[allow(clippy::too_many_arguments)]
12123async fn run_commands(
12124 state: &mut RunState,
12125 node: &str,
12126 task: &str,
12127 attempt: usize,
12128 shell: &[String],
12129 commands: &[String],
12130 cwd: &Path,
12131 timeout: Duration,
12132) -> (Vec<CommandOutcome>, Vec<u32>) {
12133 if commands.is_empty() {
12134 return (Vec::new(), Vec::new());
12139 }
12140 let mut out = Vec::new();
12141 let mut timed_out_pids = Vec::new();
12142 let total = commands.len();
12143 for (idx, command) in commands.iter().enumerate() {
12144 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12145 if let Err(e) = state.save() {
12146 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12147 }
12148 let started = Instant::now();
12149 let mut cmd = tokio::process::Command::new(&shell[0]);
12150 cmd.quiet();
12151 for key in crate::worker::INVOCATION_ENV {
12155 cmd.env_remove(key);
12156 }
12157 cmd.args(&shell[1..])
12158 .arg(command)
12159 .current_dir(cwd)
12160 .stdin(std::process::Stdio::null())
12161 .stdout(std::process::Stdio::piped())
12162 .stderr(std::process::Stdio::piped())
12163 .kill_on_drop(true);
12164 let mut outputs = Vec::new();
12165 let protected = (|| -> Result<()> {
12166 crate::incremental::configure(&mut cmd, cwd)?;
12167 let mut paths = vec![cwd.to_owned()];
12168 if let Some(cache) = state.config.cache_dir() {
12169 paths.push(cache);
12170 }
12171 outputs.extend(crate::retention::OutputLease::admit(
12172 &crate::run::home(),
12173 &paths,
12174 state.config.disk.min_free_bytes,
12175 )?);
12176 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12177 outputs.push(guard);
12178 }
12179 if let Some(cache) = state.config.cache_dir() {
12180 outputs.push(crate::retention::OutputLease::cache(
12181 &crate::run::home(),
12182 &cache,
12183 false,
12184 )?);
12185 }
12186 Ok(())
12187 })();
12188 if let Err(e) = protected {
12189 let detail = format!("execution admission unavailable: {e:#}");
12190 state.resource_block = Some(crate::run::ResourceBlock {
12191 cause: crate::run::ResourceBlockCause::Other,
12192 node: node.to_owned(),
12193 kind: if detail.contains("disk admission") {
12194 "disk admission"
12195 } else {
12196 "output ownership"
12197 }
12198 .to_owned(),
12199 detail: detail.clone(),
12200 at: Timestamp::now(),
12201 });
12202 out.push(CommandOutcome {
12203 command: command.clone(),
12204 code: None,
12205 output_tail: detail,
12206 duration_ms: 0,
12207 resource_blocked: true,
12208 });
12209 break;
12210 }
12211 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12212 let (code, body) = match spawned {
12213 Ok(mut running) => {
12214 let pid = running.child().id();
12215 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12216 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12217 let mut paths = vec![cwd.to_owned()];
12218 if let Some(cache) = state.config.cache_dir() {
12219 paths.push(cache);
12220 }
12221 let mut pressure = None;
12222 let status = tokio::select! {
12223 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12224 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12225 pressure = Some(reason); None
12226 }
12227 };
12228 let (code, failure) = match status {
12229 Some(Ok(Ok(status))) => (
12230 status.code(),
12231 status
12232 .code()
12233 .is_none()
12234 .then(|| format!("command terminated: {status}")),
12235 ),
12236 Some(Ok(Err(error))) => {
12237 (None, Some(format!("failed waiting for command: {error}")))
12238 }
12239 Some(Err(_)) => {
12240 if !running.coordinated()
12244 && let Some(pid) = pid
12245 {
12246 timed_out_pids.push(pid);
12247 }
12248 (
12249 None,
12250 Some(format!("timed out after {}s", timeout.as_secs())),
12251 )
12252 }
12253 None => (None, None),
12254 };
12255 running.finish().await;
12256 if let Some(reason) = pressure {
12257 state.resource_block = Some(crate::run::ResourceBlock {
12258 cause: crate::run::ResourceBlockCause::Other,
12259 node: node.to_owned(),
12260 kind: "disk pressure".to_owned(),
12261 detail: reason.clone(),
12262 at: Timestamp::now(),
12263 });
12264 out.push(CommandOutcome {
12265 command: command.clone(),
12266 code: None,
12267 output_tail: reason,
12268 duration_ms: started.elapsed().as_millis() as u64,
12269 resource_blocked: true,
12270 });
12271 break;
12272 }
12273 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12274 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12275 if let Some(failure) = failure {
12276 body.push('\n');
12277 body.push_str(&failure);
12278 }
12279 (code, body)
12280 }
12281 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12282 };
12283 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12284 if !body.is_empty() {
12285 match crate::run::write_verification_artifact(state, task, &body) {
12289 Ok(name) if summary != body => {
12290 summary = format!("[full output: artifacts/{name}]\n{summary}");
12291 }
12292 Ok(_) => {}
12293 Err(e) => {
12294 let warning = format!("could not save the full {task} output: {e:#}");
12295 tracing::warn!("{warning}");
12296 state.event(node, warning);
12297 if summary != body {
12298 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12299 }
12300 }
12301 }
12302 }
12303 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12304 out.push(CommandOutcome {
12305 command: command.clone(),
12306 code,
12307 output_tail: summary,
12308 duration_ms: started.elapsed().as_millis() as u64,
12309 resource_blocked,
12310 });
12311 if resource_blocked {
12312 state.resource_block = Some(crate::run::ResourceBlock {
12313 cause: crate::run::ResourceBlockCause::Other,
12314 node: node.to_owned(),
12315 kind: "incremental cache admission".into(),
12316 detail: body,
12317 at: Timestamp::now(),
12318 });
12319 break;
12320 }
12321 }
12322 state.task_finished(task);
12323 if let Err(e) = state.save() {
12324 tracing::warn!("could not persist the end of {task}: {e:#}");
12325 }
12326 (out, timed_out_pids)
12327}
12328
12329fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12341 let repo = repo.display();
12342 match style {
12343 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12344 MergeStyle::Squash => {
12345 let subject = message
12348 .lines()
12349 .next()
12350 .unwrap_or(branch)
12351 .replace(['\\', '"', '$', '`'], "");
12352 format!(
12353 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12354 )
12355 }
12356 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12357 }
12358}
12359
12360const PR_TITLE_MAX: usize = 240;
12371
12372fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12383 let line = queue::first_line(text)?;
12384 let chars: Vec<char> = line.chars().collect();
12385 let (kept, mark) = if chars.len() <= max {
12386 (line.to_owned(), "")
12387 } else if let Some(end) = sentence_end(&chars, max) {
12388 (chars[..end].iter().collect::<String>(), "")
12389 } else {
12390 let room = max.saturating_sub(3);
12391 let cut = if chars[room].is_whitespace() {
12394 room
12395 } else {
12396 chars[..room]
12397 .iter()
12398 .rposition(|c| c.is_whitespace())
12399 .unwrap_or(room)
12400 };
12401 let head: String = chars[..cut].iter().collect();
12402 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12403 (head.to_owned(), "...")
12404 };
12405 if kept.is_empty() {
12406 return None;
12407 }
12408 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12409 return None;
12410 }
12411 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12412 return None;
12413 }
12414 Some(format!("{kept}{mark}"))
12415}
12416
12417fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12421 const MIN: usize = 20;
12422 for i in MIN..max.min(chars.len()) {
12423 if !matches!(chars[i], '.' | '!' | '?') {
12424 continue;
12425 }
12426 let Some(&next) = chars.get(i + 1) else {
12427 continue;
12428 };
12429 if !next.is_whitespace() {
12430 continue;
12431 }
12432 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12433 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12434 continue;
12435 }
12436 let word: String = chars[..i]
12437 .iter()
12438 .rev()
12439 .take_while(|c| !c.is_whitespace())
12440 .collect::<Vec<_>>()
12441 .into_iter()
12442 .rev()
12443 .collect();
12444 let word = word.to_ascii_lowercase();
12445 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12446 continue;
12447 }
12448 let end = chars[..i]
12449 .iter()
12450 .rposition(|c| !c.is_whitespace())
12451 .map_or(i, |p| p + 1);
12452 return Some(end);
12453 }
12454 None
12455}
12456
12457struct PrMessage {
12462 title: String,
12463 body: String,
12464}
12465
12466impl PrMessage {
12467 fn commit_message(&self) -> String {
12471 format!("{}\n\n{}", self.title, self.body)
12472 }
12473}
12474
12475fn title_marker(line: &str) -> Option<&str> {
12477 let line = line.trim();
12478 let head = line.get(..6)?;
12479 head.eq_ignore_ascii_case("title:")
12480 .then(|| line[6..].trim())
12481}
12482
12483fn summary_title(summary: &str) -> Option<String> {
12488 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12489 let raw = title_marker(first)?;
12490 if raw.is_empty() {
12491 return None;
12492 }
12493 let title = queue::title_from(raw, PR_TITLE_MAX);
12494 let lower = title.to_ascii_lowercase();
12495 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12496 return None;
12497 }
12498 Some(title)
12499}
12500
12501const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12503
12504pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12506
12507pub(crate) fn is_review_run(state: &RunState) -> bool {
12511 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12512}
12513
12514fn review_title(state: &RunState) -> Option<String> {
12519 english_subject(
12520 state.reviewed_commits.as_ref()?.first()?,
12521 state.config.graph.github_english(),
12522 )
12523}
12524
12525fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12528 let raw = raw.trim();
12529 if raw.is_empty() {
12530 return None;
12531 }
12532 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12533 return None;
12534 }
12535 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12536 return None;
12537 }
12538 let title = queue::title_from(raw, PR_TITLE_MAX);
12539 let lower = title.to_ascii_lowercase();
12540 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12541 return None;
12542 }
12543 Some(title)
12544}
12545
12546#[derive(Debug, Clone, PartialEq, Eq)]
12549struct BranchFacts {
12550 commits: Vec<(String, String)>,
12552 stat: String,
12554}
12555
12556const STAT_FILE_LINES: usize = 25;
12558const COMMITS_MAX_CHARS: usize = 20_000;
12560
12561async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12564 let commits = git::commit_log(repo, base, branch).await.ok()?;
12565 if commits.is_empty() {
12566 return None;
12567 }
12568 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12569 let lines: Vec<&str> = stat.lines().collect();
12570 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12571 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12572 let more = format!(" ... {omitted} more file(s)");
12573 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12574 kept.push(&more);
12575 kept.push(lines[lines.len() - 1]);
12576 kept.join("\n")
12577 } else {
12578 lines.join("\n")
12579 };
12580 Some(BranchFacts { commits, stat })
12581}
12582
12583fn markdown_safe(text: &str) -> String {
12586 text.replace("</details>", "</details>")
12587 .replace("\x60\x60\x60", "~~~")
12588}
12589
12590fn neutral_title(state: &RunState, winner: char) -> String {
12591 format!(
12592 "chore: land candidate {} of run {}",
12593 winner.to_ascii_uppercase(),
12594 state.id
12595 )
12596}
12597
12598pub(crate) async fn repair_landing_gate(
12601 state: &mut RunState,
12602 winner: &Candidate,
12603 outcomes: &[CommandOutcome],
12604) -> Result<GateFix> {
12605 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12606 return Ok(GateFix::Stop);
12607 }
12608 let mut runner = Runner {
12609 roles: state.config.resolve_roles()?,
12610 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12611 state: state.clone(),
12612 pause: Pause::new(),
12613 interrupt: Pause::new(),
12614 };
12615 let result = runner.gate_fix_round(winner, outcomes).await;
12616 *state = runner.state;
12617 result
12618}
12619
12620pub(crate) async fn verify_current_head(
12623 state: &mut RunState,
12624 winner: &Candidate,
12625 wait: &crate::repo_lock::WaitPolicy,
12626) -> Result<bool> {
12627 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12628 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12630 state
12631 .verification_history
12632 .push(crate::run::VerificationEvidence {
12633 superseded_by: Some(head.clone()),
12634 archived_at: Some(Timestamp::now()),
12635 fixes_spent: Some(state.gate_fixes.len()),
12636 gate_head: None,
12637 gate_ran: false,
12638 gate: Vec::new(),
12639 e2e: state.head_e2e.take(),
12640 });
12641 }
12642 if let Some(e) = &state.head_e2e
12643 && e.head == head
12644 {
12645 if e.passed {
12646 return Ok(true);
12647 }
12648 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12649 state.status = RunStatus::Blocked;
12650 state.save()?;
12651 return Ok(false);
12652 }
12653 } else if state.reviews.last().is_some_and(|r| {
12654 r.verified_head.as_deref() == Some(head.as_str())
12655 && (r.e2e_status() == E2eStatus::Passed
12656 || (r.e2e_status() == E2eStatus::NotConfigured
12657 && state.config.verify.e2e.is_empty()))
12658 }) || state
12659 .gate_repair_e2e
12660 .as_ref()
12661 .is_some_and(|e| e.head == head && e.passed)
12662 {
12663 return Ok(true);
12664 }
12665 if let Some(e) = &state.gate_repair_e2e
12666 && e.head == head
12667 && !e.passed
12668 && !e.outcomes.is_empty()
12669 && !verify_inconclusive(&e.outcomes)
12670 {
12671 state.status = RunStatus::Blocked;
12672 state.save()?;
12673 return Ok(false);
12674 }
12675 state.head_e2e = Some(crate::run::GateRepairE2e {
12676 head: head.clone(),
12677 passed: false,
12678 outcomes: Vec::new(),
12679 });
12680 state.save()?;
12681 let commands = state.config.verify.e2e.clone();
12682 let shell = state.config.shell();
12683 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12684 let cache_dir = state.config.cache_dir();
12685 let outcomes = if commands.is_empty() {
12686 Vec::new()
12687 } else {
12688 let (outcomes, _) = with_cache_lease(
12689 state,
12690 cache_dir.as_deref(),
12691 "e2e",
12692 "e2e",
12693 &winner.worktree,
12694 &head,
12695 timeout,
12696 "current head e2e",
12697 wait,
12698 |state, budget| async move {
12699 run_e2e_with_retry(
12700 state,
12701 &shell,
12702 &commands,
12703 &winner.worktree,
12704 budget,
12705 "current head e2e",
12706 )
12707 .await
12708 },
12709 )
12710 .await?;
12711 outcomes
12712 };
12713 let unresolved = verify_inconclusive(&outcomes);
12714 let passed = !unresolved
12715 && outcomes.iter().all(CommandOutcome::ok)
12716 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12717 if !outcomes.is_empty() && !unresolved {
12718 state.record_base_progress(&head);
12719 }
12720 state.head_e2e = Some(crate::run::GateRepairE2e {
12721 head: head.clone(),
12722 passed,
12723 outcomes,
12724 });
12725 if !passed && !unresolved {
12726 state.status = RunStatus::Blocked;
12727 state.event(
12728 "e2e",
12729 format!("verify.e2e failed on head {head}; nothing published"),
12730 );
12731 }
12732 state.save()?;
12733 Ok(passed)
12734}
12735
12736pub(crate) async fn check_gate_repair_e2e(
12739 state: &mut RunState,
12740 winner: &Candidate,
12741 wait: &crate::repo_lock::WaitPolicy,
12742) -> Result<GateFix> {
12743 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12744 return Ok(GateFix::Retry);
12745 }
12746 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12747 if state
12748 .gate_repair_e2e
12749 .as_ref()
12750 .is_some_and(|r| r.head == head && r.passed)
12751 {
12752 return Ok(GateFix::Retry);
12753 }
12754 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12755 head: head.clone(),
12756 passed: false,
12757 outcomes: Vec::new(),
12758 });
12759 state.save()?;
12760 let commands = state.config.verify.e2e.clone();
12761 let outcomes = if commands.is_empty() {
12762 Vec::new()
12763 } else {
12764 let shell = state.config.shell();
12765 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12766 let cache_dir = state.config.cache_dir();
12767 let (outcomes, _) = with_cache_lease(
12768 state,
12769 cache_dir.as_deref(),
12770 "e2e",
12771 "e2e",
12772 &winner.worktree,
12773 &head,
12774 timeout,
12775 "gate repair e2e",
12776 wait,
12777 |state, budget| async move {
12778 run_e2e_with_retry(
12779 state,
12780 &shell,
12781 &commands,
12782 &winner.worktree,
12783 budget,
12784 "gate repair e2e",
12785 )
12786 .await
12787 },
12788 )
12789 .await?;
12790 outcomes
12791 };
12792 let deferred = verify_inconclusive(&outcomes);
12793 let passed = outcomes.iter().all(CommandOutcome::ok)
12794 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12795 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12796 head,
12797 passed,
12798 outcomes,
12799 });
12800 state.save()?;
12801 if deferred {
12802 Ok(GateFix::Defer)
12803 } else if !passed {
12804 state.event(
12805 "gate",
12806 "verify.e2e failed after gate repair; nothing published",
12807 );
12808 Ok(GateFix::Stop)
12809 } else {
12810 Ok(GateFix::Retry)
12811 }
12812}
12813
12814pub(crate) async fn run_landing_gate(
12816 state: &mut RunState,
12817 winner: &Candidate,
12818 wait: &crate::repo_lock::WaitPolicy,
12819) -> Result<Vec<CommandOutcome>> {
12820 let shell = state.config.shell();
12821 let gate_commands = state.config.verify.gate.clone();
12822 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12823 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12824 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12825 Vec::new()
12826 } else {
12827 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12828 let cache_dir = state.config.cache_dir();
12829 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12830 let (outcomes, _) = with_cache_lease(
12831 state,
12832 cache_dir.as_deref(),
12833 "gate",
12834 "gate",
12835 &winner.worktree,
12836 &head,
12837 timeout,
12838 "final gate",
12839 wait,
12840 |state, budget| {
12841 let shell = shell.clone();
12842 let gate_commands = gate_commands.clone();
12843 let winner = winner.clone();
12844 async move {
12845 Runner::run_pre_gate(state, &winner).await;
12846 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12848 let (outcomes, timed_out_pids) = run_commands(
12849 state,
12850 "gate",
12851 "gate",
12852 0,
12853 &shell,
12854 &gate_commands,
12855 &winner.worktree,
12856 budget,
12857 )
12858 .await;
12859 (outcomes, false, timed_out_pids)
12860 }
12861 },
12862 )
12863 .await?;
12864 outcomes
12865 };
12866 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12867 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12868 let mut outcomes = outcomes;
12869 outcomes.push(CommandOutcome {
12870 command: "gate head stability".into(), code: None,
12871 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12872 duration_ms: 0, resource_blocked: false,
12873 });
12874 return Ok(outcomes);
12875 }
12876 if outcomes.is_empty() {
12877 state.event("gate", "no gate commands configured; not tested");
12882 }
12883 for o in &outcomes {
12884 state.event(
12885 "gate",
12886 format!(
12887 "`{}` -> {}",
12888 o.command,
12889 if o.ok() {
12890 "pass".to_owned()
12891 } else {
12892 format!(
12893 "FAIL ({:?})\n{}",
12894 o.code,
12895 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12896 )
12897 }
12898 ),
12899 );
12900 }
12901 Ok(outcomes)
12902}
12903
12904pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12910 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12911 ""
12912 } else {
12913 pr_title
12914 }
12915}
12916
12917pub fn landing_subject_source(state: &RunState) -> String {
12921 if is_review_run(state) {
12922 let winner = state.candidates.first().map_or('A', |c| c.label);
12923 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12924 }
12925 state.instruction.clone()
12926}
12927
12928fn summary_without_title(summary: &str) -> String {
12931 let mut lines = summary.trim().lines().peekable();
12932 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12933 lines.next();
12934 }
12935 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12936}
12937
12938#[cfg(test)]
12950fn pr_message(state: &RunState, winner: char) -> PrMessage {
12951 pr_message_with(state, winner, None)
12952}
12953
12954#[cfg(test)]
12957fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12958 let raw = pr_message_raw(
12959 state,
12960 winner,
12961 facts,
12962 crate::pr_outcome::FinalHead::Unreported,
12963 ChangeNote::NotAsked,
12964 );
12965 let id = crate::scrub::Identity::current();
12966 PrMessage {
12967 title: crate::scrub::scrub(&raw.title, &id),
12968 body: crate::scrub::scrub(&raw.body, &id),
12969 }
12970}
12971
12972#[derive(Debug, Clone, Copy)]
12974enum ChangeNote<'a> {
12975 NotAsked,
12977 Summary(&'a crate::run::ChangeSummary),
12979 Unavailable(&'a str),
12981}
12982
12983fn pr_message_raw(
12984 state: &RunState,
12985 winner: char,
12986 facts: Option<&BranchFacts>,
12987 head: crate::pr_outcome::FinalHead<'_>,
12988 change: ChangeNote<'_>,
12989) -> PrMessage {
12990 if !matches!(change, ChangeNote::NotAsked) {
12991 let (title, mut body) = match change {
12992 ChangeNote::Summary(c) => (
12993 c.title.clone(),
12994 format!("## Summary\n\n{}\n\n", c.body.trim()),
12995 ),
12996 ChangeNote::Unavailable(why) => (
12997 neutral_title(state, winner),
12998 format!(
12999 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13000 ),
13001 ),
13002 ChangeNote::NotAsked => unreachable!(),
13003 };
13004 body.push_str(&public_pr_evidence(state, head));
13005 body.push_str(&format!(
13006 "\n---\nmagi:run/{} magi:candidate-{}\n",
13007 state.id,
13008 winner.to_ascii_lowercase()
13009 ));
13010 return PrMessage { title, body };
13011 }
13012 pr_preview(state, winner, facts, head)
13013}
13014
13015fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13018 let mut body = crate::pr_outcome::review_rounds(state);
13019 body.push_str(&crate::pr_outcome::final_verification(state, head));
13020 let open = state.last_round_findings();
13021 if !open.is_empty() {
13022 body.push_str("## Open review findings\n\n");
13023 for f in open {
13024 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13025 }
13026 body.push('\n');
13027 }
13028 if let Some(c) = state.change_summary.as_ref()
13029 && head == crate::pr_outcome::FinalHead::At(&c.head)
13030 {
13031 body.push_str(&format!(
13032 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13033 c.base, c.head
13034 ));
13035 }
13036 body
13037}
13038
13039fn pr_preview(
13041 state: &RunState,
13042 winner: char,
13043 facts: Option<&BranchFacts>,
13044 head: crate::pr_outcome::FinalHead<'_>,
13045) -> PrMessage {
13046 let summary = state
13047 .candidates
13048 .iter()
13049 .find(|c| c.label == winner)
13050 .map(|c| c.summary.as_str())
13051 .unwrap_or_default();
13052 let review = is_review_run(state);
13055 let title = if review {
13056 facts
13057 .and_then(|f| {
13058 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13059 })
13060 .or_else(|| review_title(state))
13061 .or_else(|| {
13062 state
13063 .candidates
13064 .iter()
13065 .find(|c| c.label == winner)
13066 .filter(|c| !c.branch.starts_with("magi/"))
13067 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13068 })
13069 .unwrap_or_else(|| neutral_title(state, winner))
13070 } else {
13071 summary_title(summary).unwrap_or_else(|| {
13072 english_title(
13073 &state.instruction,
13074 PR_TITLE_MAX,
13075 state.config.graph.github_english(),
13076 )
13077 .unwrap_or_else(|| neutral_title(state, winner))
13078 })
13079 };
13080
13081 let mut body = String::new();
13082 let what = summary_without_title(summary);
13083 if !what.is_empty() {
13084 body.push_str("## Summary\n\n");
13085 body.push_str(&what);
13086 body.push_str("\n\n");
13087 }
13088
13089 body.push_str(&crate::pr_outcome::review_rounds(state));
13091 body.push_str(&crate::pr_outcome::final_verification(state, head));
13092 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13093
13094 let open = state.open_findings();
13095 if !open.is_empty() {
13096 body.push_str("## Open review findings\n\n");
13097 for f in &open {
13098 body.push_str(&format!(
13099 "- `{}` [{:?}] {}\n",
13100 f.id, f.severity, "Context retained in local review records."
13101 ));
13102 }
13103 body.push('\n');
13104 }
13105
13106 if let Some(fix) = fix
13107 && !fix.rejected.is_empty()
13108 {
13109 body.push_str("## Declined by the fixer\n\n");
13110 for r in &fix.rejected {
13111 body.push_str(&format!(
13112 "- `{}`: {}\n",
13113 r.id, "Reason retained in local review records."
13114 ));
13115 }
13116 body.push('\n');
13117 }
13118
13119 if review {
13120 body.push_str("## Commits under review\n\n");
13122 if let Some(facts) = facts {
13123 let mut left = COMMITS_MAX_CHARS;
13124 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13125 let mut entry = format!("- {}\n", markdown_safe(subject));
13126 for l in markdown_safe(text).lines() {
13127 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13128 }
13129 if left == 0 {
13130 body.push_str(&format!(
13131 "- ... {} more commit(s)\n",
13132 facts.commits.len() - i
13133 ));
13134 break;
13135 }
13136 if entry.len() > left {
13137 let mut end = left;
13140 while !entry.is_char_boundary(end) {
13141 end -= 1;
13142 }
13143 entry.truncate(end);
13144 entry.push_str("\n ... (truncated)\n");
13145 left = 0;
13146 } else {
13147 left -= entry.len();
13148 }
13149 body.push_str(&entry);
13150 }
13151 if !facts.stat.trim().is_empty() {
13152 body.push_str(&format!(
13153 "\n## Diff stat\n\n```\n{}\n```\n",
13154 markdown_safe(facts.stat.trim())
13155 ));
13156 }
13157 } else {
13158 match &state.reviewed_commits {
13159 Some(subjects) => {
13160 for s in subjects {
13161 body.push_str(&format!("- {}\n", s.trim()));
13162 }
13163 }
13164 None => {
13165 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13168 body.push_str(rest.trim());
13169 body.push('\n');
13170 }
13171 }
13172 }
13173 } else {
13174 let task = state.instruction.trim();
13175 let task = if task.is_empty() {
13176 "(empty task)"
13177 } else {
13178 task
13179 };
13180 body.push_str(&format!(
13181 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13182 task.replace("</details>", "</details>")
13183 ));
13184 }
13185
13186 body.push_str(&format!(
13187 "\n---\nmagi:run/{} magi:candidate-{}\n",
13188 state.id,
13189 winner.to_ascii_lowercase()
13190 ));
13191
13192 PrMessage { title, body }
13193}
13194
13195fn seeded_instruction(state: &RunState) -> String {
13198 let mut inputs = Vec::new();
13199 if let Some(start) = &state.sources.start_from {
13200 inputs.push(format!(
13201 "Start from `{}` at `{}` (including ancestry).",
13202 start.token, start.sha
13203 ));
13204 }
13205 for pick in &state.sources.pick_commits {
13206 inputs.push(format!(
13207 "Pick `{}` at `{}` (single commit).",
13208 pick.token, pick.sha
13209 ));
13210 }
13211 if inputs.is_empty() {
13212 state.instruction.clone()
13213 } else {
13214 format!(
13215 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13216 state.instruction,
13217 inputs.join("\n")
13218 )
13219 }
13220}
13221
13222async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13226 let base = &state.base_branch;
13227 let mut against = base.clone();
13228 if mode == MergeMode::Pr {
13229 let remote = &state.config.merge.remote;
13233 let tracking = format!("{remote}/{base}");
13234 let fetched = git::fetch(repo, remote, base).await;
13235 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13236 against = tracking;
13237 } else {
13238 return false;
13239 }
13240 }
13241 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13242}
13243
13244fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13247 let mut detail = format!(
13248 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13249 nothing to open a pull request for"
13250 );
13251 detail.push_str(
13252 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13253 );
13254 detail
13255}
13256
13257#[derive(Debug, PartialEq, Eq)]
13260enum PrPlan {
13261 Create,
13262 Adopt { url: String, title: String },
13263 Stop(String),
13264}
13265
13266fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13269 match found {
13270 Ok(land::OpenPr::None) => PrPlan::Create,
13271 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13272 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13273 "several open pull requests exist for this branch, not picking one: {}",
13274 urls.join(" ")
13275 )),
13276 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13277 }
13278}
13279
13280pub(crate) async fn gh_pr_create(
13282 cwd: &Path,
13283 base: &str,
13284 head: &str,
13285 title: &str,
13286 body: &str,
13287) -> Result<String> {
13288 let out = tokio::process::Command::new("gh")
13289 .args([
13290 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13291 ])
13292 .current_dir(cwd)
13293 .quiet()
13294 .stdin(std::process::Stdio::null())
13295 .output()
13296 .await
13297 .context("spawn gh")?;
13298 if out.status.success() {
13299 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13300 } else {
13301 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13302 }
13303}
13304
13305async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13307 match std::fs::symlink_metadata(path) {
13308 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13309 git::worktree_remove_clean(repo, path).await?;
13312 return Ok(());
13313 }
13314 Err(e) => return Err(e.into()),
13315 Ok(_) => {}
13316 }
13317 if !git::is_clean(path).await? {
13318 bail!(
13319 "worktree has uncommitted work; refusing removal: {}",
13320 path.display()
13321 );
13322 }
13323 if !git::worktree_remove_clean(repo, path).await? {
13324 bail!("worktree removal refused: {}", path.display());
13325 }
13326 Ok(())
13327}
13328
13329pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13338 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13339 bail!("refusing fold: {reason}");
13340 }
13341 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13342 bail!("candidate preservation incomplete; refusing cleanup");
13343 }
13344 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13348 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13349 let _driver = crate::driver::maintenance(home, &state.id)?;
13350 let mut output_guards = Vec::new();
13351 for entry in std::fs::read_dir(state.worktree_root())
13352 .into_iter()
13353 .flatten()
13354 .flatten()
13355 {
13356 if entry.path().join(".git").is_file() {
13357 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13358 }
13359 }
13360 let repo = state.repo.clone();
13361 let wait = repository_wait(state, None);
13364 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13365 Ok(lease) => lease,
13366 Err(e) => {
13367 state.save_under(home)?;
13368 return Err(e);
13369 }
13370 };
13371 let _source_outputs = if state.status == RunStatus::Merged
13374 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13375 {
13376 verify_merged_fold_sources(state, drop_winner, home).await?
13377 } else {
13378 Vec::new()
13379 };
13380 let root = state.worktree_root();
13381 let winner = state.tally.as_ref().map(|t| t.winner);
13382 let mut removed = Vec::new();
13383
13384 for i in 0..state.candidates.len() {
13385 let c = state.candidates[i].clone();
13386 let is_winner = Some(c.label) == winner;
13387 if is_winner && !drop_winner {
13388 continue;
13389 }
13390 let existed = c.worktree.exists();
13391 remove_preserved_worktree(&repo, &c.worktree).await?;
13394 if existed {
13395 removed.push(c.worktree.to_string_lossy().into_owned());
13396 }
13397 let handed_over = state.released_branches.contains(&c.branch);
13400 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13401 if !git::branch_delete(&repo, &c.branch).await? {
13402 bail!("candidate branch removal refused");
13403 }
13404 removed.push(c.branch.clone());
13405 }
13406 state.candidates[i].folded = true;
13407 }
13408
13409 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13410 let path = name.path();
13411 let keep = !drop_winner
13412 && winner.is_some_and(|w| {
13413 path.file_name()
13414 .is_some_and(|n| n == format!("cand-{w}").as_str())
13415 });
13416 if keep {
13417 continue;
13418 }
13419 remove_preserved_worktree(&repo, &path).await?;
13420 removed.push(path.to_string_lossy().into_owned());
13421 }
13422
13423 remove_if_empty(&root);
13432
13433 if state.enabled_worktree_config && drop_winner {
13434 git::release_worktree_config(&repo).await.ok();
13438 state.enabled_worktree_config = false;
13439 }
13440 state.save_under(home)?;
13441 Ok(removed)
13442}
13443
13444async fn verify_merged_fold_sources(
13445 state: &RunState,
13446 drop_winner: bool,
13447 home: &Path,
13448) -> Result<Vec<std::fs::File>> {
13449 let winner = state.tally.as_ref().map(|t| t.winner);
13450 let mut heads = BTreeSet::new();
13451 for candidate in &state.candidates {
13452 if Some(candidate.label) == winner && !drop_winner {
13453 continue;
13454 }
13455 anyhow::ensure!(
13456 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13457 "candidate recovery source is outside the inventoried run root: {}",
13458 candidate.worktree.display()
13459 );
13460 match std::fs::symlink_metadata(&candidate.worktree) {
13461 Ok(_) => {}
13462 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13463 Err(error) => return Err(error.into()),
13464 }
13465 if !state.released_branches.contains(&candidate.branch)
13466 && git::branch_exists(&state.repo, &candidate.branch).await?
13467 {
13468 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13469 }
13470 }
13471 let mut cargo_guards = Vec::new();
13472 let entries = match std::fs::read_dir(state.worktree_root()) {
13473 Ok(entries) => Some(entries),
13474 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13475 Err(error) => return Err(error.into()),
13476 };
13477 for entry in entries.into_iter().flatten() {
13478 let path = entry?.path();
13479 let metadata = std::fs::symlink_metadata(&path)?;
13480 anyhow::ensure!(
13481 metadata.is_dir() && !metadata.file_type().is_symlink(),
13482 "non-directory or symlink recovery source retained: {}",
13483 path.display()
13484 );
13485 if !drop_winner
13486 && winner.is_some_and(|w| {
13487 path.file_name()
13488 .is_some_and(|name| name == format!("cand-{w}").as_str())
13489 })
13490 {
13491 continue;
13492 }
13493 anyhow::ensure!(
13494 path.join(".git").is_file(),
13495 "unregistered recovery source retained: {}",
13496 path.display()
13497 );
13498 anyhow::ensure!(
13499 git::git(
13500 &path,
13501 &[
13502 "status",
13503 "--porcelain",
13504 "--untracked-files=all",
13505 "--ignored",
13506 "--",
13507 ".",
13508 ":(exclude)target"
13509 ]
13510 )
13511 .await?
13512 .is_empty(),
13513 "source changes or ignored recovery files retained: {}",
13514 path.display()
13515 );
13516 match std::fs::symlink_metadata(path.join("target")) {
13517 Ok(_) => {
13518 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13519 }
13520 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13521 Err(error) => return Err(error.into()),
13522 }
13523 for checkpoint in [
13524 "rebase-merge",
13525 "rebase-apply",
13526 "sequencer",
13527 "CHERRY_PICK_HEAD",
13528 "MERGE_HEAD",
13529 "REBASE_HEAD",
13530 "REVERT_HEAD",
13531 "BISECT_LOG",
13532 "AUTO_MERGE",
13533 "MERGE_MSG",
13534 ] {
13535 let location = git::git(
13536 &path,
13537 &[
13538 "rev-parse",
13539 "--path-format=absolute",
13540 "--git-path",
13541 checkpoint,
13542 ],
13543 )
13544 .await?;
13545 anyhow::ensure!(
13546 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13547 if e.kind() == std::io::ErrorKind::NotFound),
13548 "Git recovery checkpoint retained: {checkpoint}"
13549 );
13550 }
13551 heads.insert(git::rev_parse(&path, "HEAD").await?);
13552 }
13553 crate::fold_evidence::preserve(state, home, &heads).await?;
13554
13555 Ok(cargo_guards)
13556}
13557
13558fn remove_if_empty(dir: &Path) {
13569 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13570 std::fs::remove_dir(dir).ok();
13571 }
13572}
13573
13574pub fn worst_open(state: &RunState) -> Option<Severity> {
13576 state
13577 .reviews
13578 .last()?
13579 .reviews
13580 .iter()
13581 .flat_map(|r| r.findings.iter())
13582 .map(|f| f.severity)
13583 .max()
13584}
13585
13586async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13590 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13591 git::acquire_worktree_config(repo).await?;
13592 state.enabled_worktree_config = true;
13593 }
13594 Ok(())
13595}
13596
13597#[cfg(test)]
13598mod tests {
13599 #[tokio::test]
13600 async fn launcher_base_fetch_requires_remote_success() {
13601 let tmp = tempfile::tempdir().unwrap();
13602 let repo = tmp.path().join("repo");
13603 let remote = tmp.path().join("remote");
13604 std::fs::create_dir(&repo).unwrap();
13605 std::fs::create_dir(&remote).unwrap();
13606 git::git(&remote, &["init", "--bare", "-b", "main"])
13607 .await
13608 .unwrap();
13609 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13610 git::git(
13611 &repo,
13612 &[
13613 "-c",
13614 "user.name=Fixture",
13615 "-c",
13616 "user.email=fixture@example.invalid",
13617 "commit",
13618 "--allow-empty",
13619 "-m",
13620 "fixture",
13621 ],
13622 )
13623 .await
13624 .unwrap();
13625 git::git(
13626 &repo,
13627 &["remote", "add", "origin", remote.to_str().unwrap()],
13628 )
13629 .await
13630 .unwrap();
13631 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13632 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13633 assert_eq!(
13634 resolve_base(&repo, "main", "origin", None)
13635 .await
13636 .unwrap()
13637 .commit,
13638 head
13639 );
13640 #[cfg(unix)]
13641 {
13642 let policy: crate::config::Coordination = toml::from_str(
13643 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13644 ).unwrap();
13645 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13646 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13647 let held = crate::repo_lock::SeatLease::operation_with(
13648 &repo,
13649 Some(&policy),
13650 &crate::repo_lock::WaitPolicy::default(),
13651 )
13652 .await
13653 .unwrap();
13654 assert!(held.is_some());
13655 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13656 tokio::pin!(fetch);
13657 assert!(
13658 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13659 .await
13660 .is_err()
13661 );
13662 assert!(
13663 !repo.join(".git/FETCH_HEAD").exists(),
13664 "no fetch before admission"
13665 );
13666 drop(held);
13667 let fetched = fetch.await.unwrap();
13668 assert_eq!(fetched.commit, head);
13669 assert_eq!(fetched.remote, "origin");
13670 assert_eq!(fetched.branch, "main");
13671 }
13672 git::git(
13674 &repo,
13675 &[
13676 "remote",
13677 "set-url",
13678 "origin",
13679 tmp.path().join("missing").to_str().unwrap(),
13680 ],
13681 )
13682 .await
13683 .unwrap();
13684 let error = resolve_base(&repo, "main", "origin", None)
13685 .await
13686 .unwrap_err()
13687 .to_string();
13688 assert!(error.contains("cannot read origin/main"));
13689 assert!(error.contains("refusing to branch off"));
13690 }
13691
13692 #[test]
13693 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13694 let leaked = vec!["chore(deps): update a crate".to_owned()];
13695 let own = "fix(daemon): apply a chosen action";
13696 assert!(should_retitle("", own, &leaked));
13697 assert!(should_retitle(
13698 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13699 own,
13700 &leaked
13701 ));
13702 assert!(should_retitle(
13703 "chore: land candidate A of run 1",
13704 own,
13705 &leaked
13706 ));
13707 assert!(should_retitle(
13708 "magi: candidate A (uncommitted work)",
13709 own,
13710 &leaked
13711 ));
13712 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13713 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13714 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13715 }
13716
13717 #[test]
13718 fn pr_merge_plan_creates_adopts_or_stops() {
13719 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13720 assert_eq!(
13721 pr_merge_plan(Ok(land::OpenPr::One {
13722 url: "u".into(),
13723 title: "t".into()
13724 })),
13725 PrPlan::Adopt {
13726 url: "u".into(),
13727 title: "t".into()
13728 }
13729 );
13730 let PrPlan::Stop(many) =
13731 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13732 else {
13733 panic!("many must stop");
13734 };
13735 assert!(many.contains('a') && many.contains('b'));
13736 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13737 panic!("a failed lookup must stop");
13738 };
13739 assert!(err.contains("bad token"));
13740 }
13741
13742 use super::*;
13743 use crate::run::GateStatus;
13744 use std::collections::BTreeMap;
13745 use std::time::Duration;
13746
13747 fn conductor() -> AgentSpec {
13748 AgentSpec {
13749 id: "conductor".to_owned(),
13750 kind: crate::config::AgentKind::Command,
13751 model: None,
13752 command: vec!["true".to_owned()],
13753 extra_args: Vec::new(),
13754 env: BTreeMap::new(),
13755 prompt_delivery: None,
13756 max_turn_secs: None,
13757 }
13758 }
13759
13760 fn spec(id: &str) -> AgentSpec {
13761 AgentSpec {
13762 id: id.to_owned(),
13763 kind: crate::config::AgentKind::Command,
13764 model: None,
13765 command: vec!["true".to_owned()],
13766 extra_args: Vec::new(),
13767 env: BTreeMap::new(),
13768 prompt_delivery: None,
13769 max_turn_secs: None,
13770 }
13771 }
13772
13773 fn ids(xs: &[&str]) -> BTreeSet<String> {
13774 xs.iter().map(|s| (*s).to_owned()).collect()
13775 }
13776
13777 #[test]
13778 fn pick_successor_skips_an_agent_another_seat_holds() {
13779 let roster = [spec("a"), spec("b"), spec("c")];
13781 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13782 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13783 }
13784
13785 #[test]
13786 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13787 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13789 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13790 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13791 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13792 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13793 }
13794
13795 #[test]
13796 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13797 let roster = [spec("a"), spec("b")];
13798 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13799 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13800 let carried = ids(&["b"]);
13801 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13802 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13803 }
13804
13805 #[test]
13806 fn pick_successor_returns_none_without_any_untried_successor() {
13807 let roster = [spec("a"), spec("b")];
13808 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13809 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13810 let carried = ids(&["a"]);
13811 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13812 }
13813
13814 #[test]
13815 fn pick_successor_rescue_avoids_another_seats_occupant() {
13816 let roster = [spec("a"), spec("b"), spec("c")];
13818 let carried = ids(&["b", "c"]);
13819 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13820 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13821 }
13822
13823 #[test]
13824 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13825 let roster = [spec("a"), spec("b"), spec("c")];
13826 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13827 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13828 }
13829
13830 #[test]
13831 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13832 let roster = [spec("a"), spec("b"), spec("c")];
13833 let failed = ids(&["a", "b", "c"]);
13834 let mut tried = ids(&["b"]);
13836 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13837 assert_eq!(first.id, "a");
13838 tried.insert(first.id.clone());
13839 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13840 assert_eq!(second.id, "c");
13841 tried.insert(second.id.clone());
13842 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13843 }
13844
13845 #[test]
13846 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13847 let roster = [spec("a"), spec("b")];
13848 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13849 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13850 }
13851
13852 #[test]
13853 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13854 let roster = [spec("a"), spec("b"), spec("c")];
13855 let h = SeatHistory {
13856 failed: ids(&["a"]),
13857 last_ok: Some("b".to_owned()),
13858 last_fail: None,
13859 };
13860 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13861 let gone = SeatHistory {
13863 last_ok: Some("zzz".to_owned()),
13864 ..h.clone()
13865 };
13866 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13867 let failed = SeatHistory {
13868 failed: ids(&["a", "b"]),
13869 last_ok: Some("b".to_owned()),
13870 last_fail: None,
13871 };
13872 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13873 }
13874
13875 #[test]
13876 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13877 let first = SeatState::new("review-1", "alpha", 7);
13878 let next = handover_seat("review-1", "gamma", 7);
13879 assert_ne!(first.claude_session, next.claude_session);
13880 }
13881
13882 #[test]
13883 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13884 let mut state = state_with_summary("x", "y");
13885 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13886 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13887 assert_ne!(a.claude_session, b.claude_session);
13888 }
13889
13890 #[test]
13891 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13892 let roster = [spec("a"), spec("b")];
13893 let h = SeatHistory {
13894 failed: ids(&["a", "b"]),
13895 ..SeatHistory::default()
13896 };
13897 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13898 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13899 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13900 }
13901
13902 #[test]
13909 fn failure_signature_ignores_numbers_and_paths() {
13910 assert_eq!(
13911 failure_signature("exited with Some(2) and no usable output"),
13912 failure_signature("exited with Some(137) and no usable output")
13913 );
13914 assert_eq!(
13915 failure_signature("process exited with no usable output (exit code 2)"),
13916 failure_signature("process exited with no usable output (exit code 137)")
13917 );
13918 for (old, new) in [
13919 (
13920 "exited with Some(2) and no usable output",
13921 "process exited with no usable output (exit code 2)",
13922 ),
13923 (
13924 "exited with None and no usable output",
13925 "process exited with no usable output (exit code unavailable)",
13926 ),
13927 ] {
13928 let previous = FailClass::Other(failure_signature(old));
13929 let current = FailClass::Other(failure_signature(new));
13930 assert_eq!(previous, current, "resumption retains the failure class");
13931 assert!(!should_hand_over(Some(&previous), ¤t));
13932 }
13933 assert_eq!(
13934 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13935 failure_signature("cannot open /var/x.txt: denied")
13936 );
13937 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13938 }
13939
13940 #[test]
13941 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13942 let other = FailClass::Other("x".into());
13943 assert!(should_hand_over(None, &FailClass::Quota));
13944 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13945 assert!(should_hand_over(
13946 Some(&FailClass::Timeout),
13947 &FailClass::Timeout
13948 ));
13949 assert!(should_hand_over(None, &other));
13950 assert!(!should_hand_over(Some(&other), &other));
13951 assert!(should_hand_over(
13952 Some(&other),
13953 &FailClass::Other("y".into())
13954 ));
13955 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13957 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13958 }
13959
13960 #[test]
13961 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13962 let a = SeatState::new("judge-1", "alpha", 7);
13963 let b = handover_seat("judge-1", "beta", 7);
13964 assert_ne!(a.claude_session, b.claude_session);
13965 assert_eq!(b.turns, 0);
13966 }
13967
13968 #[test]
13969 fn a_timeout_is_classified_apart_from_other_failures() {
13970 assert_eq!(
13971 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13972 Some(FailClass::Timeout)
13973 );
13974 assert!(matches!(
13975 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13976 Some(FailClass::Other(_))
13977 ));
13978 }
13979
13980 #[test]
13981 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13982 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13983 let tried = BTreeSet::from(["beta".to_owned()]);
13984 let next = next_untried_in_roster(&roster, 1, &tried);
13987 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13988 }
13989
13990 #[test]
13991 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13992 let roster = vec![spec("alpha"), spec("beta")];
13993 let tried = BTreeSet::from(["beta".to_owned()]);
13994 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13998 }
13999
14000 #[test]
14001 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14002 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14003 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14004 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14008 }
14009
14010 #[test]
14011 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14012 let roster = vec![spec("a"), spec("a"), spec("b")];
14013 let tried = BTreeSet::from(["a".to_owned()]);
14014 let next = next_untried_in_roster(&roster, 0, &tried);
14015 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14016 }
14017
14018 #[test]
14019 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14020 let roster = vec![spec("a"), spec("b")];
14021 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14022 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14023 }
14024
14025 #[test]
14026 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14027 let dir = tempfile::tempdir().unwrap();
14028 let bay = dir.path().join("ffff");
14029
14030 remove_if_empty(&bay);
14032 assert!(!bay.exists());
14033
14034 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14037 remove_if_empty(&bay);
14038 assert!(bay.exists(), "non-empty directory must survive");
14039
14040 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14042 remove_if_empty(&bay);
14043 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14044 }
14045
14046 #[test]
14055 fn a_full_panel_that_found_nothing_is_clean() {
14056 assert!(round_is_clean(
14057 0,
14058 true,
14059 2,
14060 2,
14061 0,
14062 IncompleteReviewPolicy::Block
14063 ));
14064 }
14065
14066 #[test]
14067 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14068 assert!(!round_is_clean(
14069 0,
14070 true,
14071 1,
14072 2,
14073 0,
14074 IncompleteReviewPolicy::Block
14075 ));
14076 }
14077
14078 #[test]
14079 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14080 assert!(!round_is_clean(
14081 1,
14082 true,
14083 1,
14084 2,
14085 0,
14086 IncompleteReviewPolicy::Warn
14087 ));
14088 }
14089
14090 #[test]
14091 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14092 assert!(round_is_clean(
14093 0,
14094 true,
14095 1,
14096 2,
14097 0,
14098 IncompleteReviewPolicy::Warn
14099 ));
14100 }
14101
14102 #[test]
14103 fn a_full_panel_with_an_open_finding_is_not_clean() {
14104 assert!(!round_is_clean(
14105 1,
14106 true,
14107 2,
14108 2,
14109 0,
14110 IncompleteReviewPolicy::Block
14111 ));
14112 }
14113
14114 #[test]
14115 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14116 assert!(!round_is_clean(
14117 0,
14118 false,
14119 2,
14120 2,
14121 0,
14122 IncompleteReviewPolicy::Block
14123 ));
14124 }
14125
14126 #[test]
14133 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14134 assert!(round_is_clean(
14137 0,
14138 true,
14139 1,
14140 2,
14141 1,
14142 IncompleteReviewPolicy::Block
14143 ));
14144 }
14145
14146 #[test]
14147 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14148 assert!(!round_is_clean(
14151 0,
14152 true,
14153 1,
14154 2,
14155 0,
14156 IncompleteReviewPolicy::Block
14157 ));
14158 }
14159
14160 #[test]
14161 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14162 assert!(!round_is_clean(
14163 1,
14164 true,
14165 1,
14166 2,
14167 1,
14168 IncompleteReviewPolicy::Block
14169 ));
14170 assert!(!round_is_clean(
14171 0,
14172 false,
14173 1,
14174 2,
14175 1,
14176 IncompleteReviewPolicy::Block
14177 ));
14178 }
14179
14180 #[test]
14181 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14182 assert!(!round_is_clean(
14186 0,
14187 true,
14188 0,
14189 2,
14190 2,
14191 IncompleteReviewPolicy::Block
14192 ));
14193 }
14194
14195 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14196 CommandOutcome {
14197 command: "test".to_owned(),
14198 code,
14199 output_tail: String::new(),
14200 duration_ms: 0,
14201 resource_blocked,
14202 }
14203 }
14204
14205 #[cfg(unix)]
14206 #[tokio::test]
14207 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14208 ask_test_home();
14209 let temp = tempfile::tempdir().unwrap();
14210 let mut runner = runner_at(RunStatus::Gating);
14211 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14212 let commands = vec![
14213 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14214 "touch must-not-run".to_owned(),
14215 ];
14216 let (outcomes, retried, pids) = run_e2e_with_retry(
14217 &mut runner.state,
14218 &shell,
14219 &commands,
14220 temp.path(),
14221 Duration::from_secs(5),
14222 "fixture",
14223 )
14224 .await;
14225 assert_eq!(outcomes.len(), 1);
14226 assert!(outcomes[0].resource_blocked);
14227 assert!(!outcomes[0].build_failed());
14228 assert!(!retried);
14229 assert!(pids.is_empty());
14230 assert!(!temp.path().join("must-not-run").exists());
14231 assert_eq!(
14232 runner.state.resource_block.as_ref().unwrap().kind,
14233 "incremental cache admission"
14234 );
14235 }
14236
14237 #[test]
14238 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14239 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14240 assert!(
14241 !verify_inconclusive(&[outcome(Some(1), false)]),
14242 "an ordinary failure is still evidence about the patch"
14243 );
14244 assert!(verify_inconclusive(&[outcome(None, true)]));
14245 assert!(
14246 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14247 "one inconclusive outcome taints the whole batch"
14248 );
14249 assert!(!verify_inconclusive(&[]));
14250 }
14251
14252 #[tokio::test]
14253 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14254 let calls = std::sync::atomic::AtomicUsize::new(0);
14258 let started = Instant::now();
14259 wait_for_pids_with(
14260 &[123],
14261 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14262 Duration::from_millis(5),
14263 Duration::from_secs(5),
14264 )
14265 .await;
14266 assert!(
14267 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14268 "must keep checking rather than deciding on the first answer"
14269 );
14270 assert!(
14271 started.elapsed() < Duration::from_secs(1),
14272 "must return the moment it is confirmed dead, not wait out the ceiling"
14273 );
14274 }
14275
14276 #[tokio::test]
14277 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14278 let started = Instant::now();
14279 wait_for_pids_with(
14280 &[123],
14281 |_| true, Duration::from_millis(5),
14283 Duration::from_millis(30),
14284 )
14285 .await;
14286 let elapsed = started.elapsed();
14287 assert!(
14288 elapsed >= Duration::from_millis(30),
14289 "must not give up before its own ceiling: {elapsed:?}"
14290 );
14291 assert!(
14292 elapsed < Duration::from_secs(1),
14293 "must not wait past its own ceiling either: {elapsed:?}"
14294 );
14295 }
14296
14297 #[tokio::test]
14298 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14299 let started = Instant::now();
14300 wait_for_pids_with(
14301 &[],
14302 |_| true,
14303 Duration::from_secs(5),
14304 Duration::from_secs(5),
14305 )
14306 .await;
14307 assert!(
14308 started.elapsed() < Duration::from_millis(200),
14309 "an empty pid list has nothing to confirm"
14310 );
14311 }
14312
14313 fn review_round(
14319 clean: bool,
14320 blocking: usize,
14321 answered: usize,
14322 expected: usize,
14323 progressed: bool,
14324 e2e_ok: bool,
14325 ) -> ReviewRound {
14326 ReviewRound {
14327 round: 1,
14328 head: "h".to_owned(),
14329 verified_head: None,
14330 verified_at: None,
14331 reviews: Vec::new(),
14332 e2e: vec![CommandOutcome {
14333 command: "test".to_owned(),
14334 code: Some(if e2e_ok { 0 } else { 1 }),
14335 output_tail: String::new(),
14336 duration_ms: 0,
14337 resource_blocked: false,
14338 }],
14339 verify_retried: false,
14340 e2e_deferred: false,
14341 e2e_defer_reason: None,
14342 fix: None,
14343 blocking,
14344 answered,
14345 expected,
14346 clean,
14347 progressed,
14348 vote_split: false,
14349 reconsideration: Vec::new(),
14350 verdict: None,
14351 }
14352 }
14353
14354 #[test]
14355 fn review_conclusion_is_none_when_nothing_has_run() {
14356 assert_eq!(review_conclusion(&[], 3), None);
14357 }
14358
14359 #[test]
14360 fn review_conclusion_is_none_while_rounds_remain() {
14361 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14362 assert_eq!(review_conclusion(&rounds, 3), None);
14363 }
14364
14365 #[test]
14366 fn review_conclusion_is_gating_once_a_round_is_clean() {
14367 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14368 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14369 }
14370
14371 #[test]
14372 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14373 let rounds = vec![
14374 review_round(false, 1, 2, 2, true, true),
14375 review_round(false, 1, 2, 2, true, true),
14376 ];
14377 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14378 }
14379
14380 #[test]
14381 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14382 let rounds = vec![
14383 review_round(false, 1, 2, 2, true, true),
14384 review_round(false, 1, 2, 2, true, false),
14385 ];
14386 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14387 }
14388
14389 #[test]
14390 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14391 let mut blocked = review_round(false, 1, 2, 2, true, false);
14398 blocked.e2e[0].resource_blocked = true;
14399 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14400 assert_eq!(review_conclusion(&rounds, 2), None);
14401 }
14402
14403 #[test]
14404 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14405 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14407 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14408 }
14409
14410 #[test]
14411 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14412 let rounds = vec![
14413 review_round(false, 1, 2, 2, false, true),
14414 review_round(false, 1, 2, 2, false, true),
14415 ];
14416 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14417 }
14418
14419 fn secs(n: u64) -> Duration {
14420 Duration::from_secs(n)
14421 }
14422
14423 #[tokio::test]
14426 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14427 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14428 if std::env::var_os(CHILD).is_none() {
14429 let output = std::process::Command::new(std::env::current_exe().unwrap())
14433 .args([
14434 "--exact",
14435 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14436 "--nocapture",
14437 ])
14438 .env(CHILD, "1")
14439 .env_remove("MAGI_WORKER_HOME")
14440 .env_remove("MAGI_WORKER_ID")
14441 .output()
14442 .unwrap();
14443 assert!(
14444 output.status.success()
14445 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14446 "{}\n{}",
14447 String::from_utf8_lossy(&output.stdout),
14448 String::from_utf8_lossy(&output.stderr)
14449 );
14450 return;
14451 }
14452 ask_test_home();
14453 let tmp = tempfile::tempdir().unwrap();
14454 let repo = tmp.path().join("repo");
14455 std::fs::create_dir_all(&repo).unwrap();
14456 init_repo(&repo);
14457 let g = |args: &[&str]| {
14458 let out = std::process::Command::new("git")
14459 .args(args)
14460 .current_dir(&repo)
14461 .quiet()
14462 .output()
14463 .unwrap();
14464 assert!(
14465 out.status.success(),
14466 "{}",
14467 String::from_utf8_lossy(&out.stderr)
14468 );
14469 };
14470 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14471 g(&["checkout", "-b", "historical"]);
14472 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14473 g(&["add", "."]);
14474 g(&["commit", "-m", "old alternative"]);
14475 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14476 g(&["checkout", "-b", "magi/task-selected", &base]);
14477 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14478 g(&["add", "."]);
14479 g(&["commit", "-m", "measurement infrastructure"]);
14480 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14481 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14482 g(&["add", "."]);
14483 g(&["commit", "-m", "dependent implementation"]);
14484 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14485 g(&["checkout", "main"]);
14486 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14487 g(&["add", "."]);
14488 g(&["commit", "-m", "unrelated base change"]);
14489 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14490 let origin = tmp.path().join("origin.git");
14491 g(&[
14492 "clone",
14493 "--bare",
14494 repo.to_str().unwrap(),
14495 origin.to_str().unwrap(),
14496 ]);
14497 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14498 g(&["fetch", "origin"]);
14499 let selection = crate::pr_recovery::Selection {
14500 url: "https://example.test/project/pull/1".into(),
14501 branch: "magi/task-selected".into(),
14502 head: head.clone(),
14503 source_head: None,
14504 source_run: None,
14505 source_proof: None,
14506 disposition: Some(crate::pr_recovery::Disposition::Review),
14507 instruction: "preserve measured behavior".into(),
14508 approved_at: jiff::Timestamp::now(),
14509 };
14510 let mut template = runner_at(RunStatus::Reviewing);
14511 template.state.repo = repo.clone();
14512 template.state.base_commit = current.clone();
14513 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14514 template.state.origin = Some(crate::run::Origin::queue("selected"));
14515 template.state.pr_head = Some(selection.branch.clone());
14516 template.state.pr_recovery = Some(selection.clone());
14517 let mut runner =
14518 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14519 .await
14520 .unwrap();
14521 let wt = runner.state.winner().unwrap().worktree.clone();
14522 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14523 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14524 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14525 assert!(runner.state.seeds.is_empty());
14526 assert_eq!(runner.state.instruction, "preserve measured behavior");
14527 runner.sync_to_base().await.unwrap();
14528 assert_eq!(
14529 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14530 "prerequisite"
14531 );
14532 assert_eq!(
14533 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14534 "measured implementation"
14535 );
14536 assert_eq!(
14537 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14538 "current main change"
14539 );
14540 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14541 assert_eq!(
14542 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14543 head
14544 );
14545
14546 let mut partial = runner_at(RunStatus::Reviewing);
14548 partial.state.repo = repo.clone();
14549 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14550 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14551 run: Some("exit 1".into()),
14552 ..Default::default()
14553 }];
14554 let selection = runner.state.pr_recovery.clone().unwrap();
14555 partial.state.pr_recovery = Some(selection.clone());
14556 partial.state.pr_head = Some(selection.branch.clone());
14557 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14558 let id = partial.state.id.clone();
14559 let error = match Runner::open_selected(
14560 &repo,
14561 partial.state,
14562 partial.roles,
14563 1,
14564 runner.state.base_commit.clone(),
14565 selection,
14566 )
14567 .await
14568 {
14569 Ok(_) => panic!("setup must fail"),
14570 Err(e) => e,
14571 };
14572 assert_eq!(
14573 error
14574 .downcast_ref::<crate::run::StartupFailure>()
14575 .unwrap()
14576 .run,
14577 id
14578 );
14579 let failed = RunState::load(&id).unwrap();
14580 assert_eq!(failed.status, RunStatus::Failed);
14581 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14582 assert!(Runner::resume(&id).is_err());
14583 runner.state.status = RunStatus::Ready;
14584 runner.state.driver_exited = true;
14585 runner.state.save().unwrap();
14586 let mut retry = runner_at(RunStatus::Reviewing);
14587 retry.state.repo = repo.clone();
14588 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14589 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14590 reselected.approved_at = jiff::Timestamp::now();
14591 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14592 retry.state.pr_head = Some(reselected.branch.clone());
14593 retry.state.pr_recovery = Some(reselected.clone());
14594 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14595 .await
14596 .unwrap();
14597 let restarted = Runner::open_selected(
14598 &repo,
14599 retry.state,
14600 retry.roles,
14601 1,
14602 runner.state.base_commit.clone(),
14603 reselected,
14604 )
14605 .await
14606 .unwrap();
14607 assert_eq!(
14608 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14609 .await
14610 .unwrap(),
14611 head
14612 );
14613 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14614 assert_eq!(
14615 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14616 head
14617 );
14618 }
14619
14620 #[tokio::test]
14621 #[cfg(unix)]
14622 async fn fallback_and_interrupted_restoration_require_coordination() {
14623 use std::os::fd::AsRawFd as _;
14624 ask_test_home();
14625 let tmp = tempfile::tempdir().unwrap();
14626 let mut runner = registered_runner(tmp.path()).await;
14627 runner.state.config.graph.timeout_resource_wait = 1;
14628 let winner = runner.state.candidates[0].clone();
14629 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14630 let leftover = winner.worktree.join("leftover");
14631 std::fs::write(&leftover, "untrusted edits").unwrap();
14632 let next = tmp.path().join("next-fixer");
14633 let mut failed = spec("first");
14634 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14635 let mut successor = spec("second");
14636 successor.command = vec![
14637 "sh".into(),
14638 "-c".into(),
14639 format!("touch '{}'; echo done", next.display()),
14640 ];
14641 let attempts = vec![
14642 (failed.clone(), "first".into()),
14643 (successor.clone(), "second".into()),
14644 ];
14645 let lock_path = runner.state.repo.join(".git/updater.lock");
14646 let external = std::fs::File::create(&lock_path).unwrap();
14647 assert_eq!(
14648 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14649 0
14650 );
14651 let artifacts = tmp.path().join("artifacts");
14652 let build = |spec, seat| SeatJob {
14653 spec,
14654 seat,
14655 cwd: winner.worktree.clone(),
14656 prompt: "fixture".into(),
14657 timeout: Duration::from_secs(10),
14658 allow_write: true,
14659 sessions: false,
14660 artifacts: artifacts.clone(),
14661 stem: "fix".into(),
14662 handover: None,
14663 };
14664 let (_, _, out) = runner
14665 .ask_fixer_among(
14666 attempts.clone(),
14667 "graft",
14668 None,
14669 Some((&winner.worktree, &head)),
14670 build,
14671 )
14672 .await;
14673 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14674 assert!(leftover.exists(), "contention must prevent clean");
14675 assert!(!next.exists());
14676 assert!(runner.state.handovers.is_empty());
14677 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14679 from_run: "earlier".into(),
14680 generation: 1,
14681 mode: crate::recovery::Mode::VerifyRepair,
14682 reasons: Vec::new(),
14683 reviewed_head: None,
14684 rounds: Vec::new(),
14685 open_findings: Vec::new(),
14686 evidence: None,
14687 criteria_inherited: true,
14688 repair: crate::recovery::Repair::Started {
14689 before: head.clone(),
14690 },
14691 });
14692 assert!(
14693 runner
14694 .recovery_repair(&winner)
14695 .await
14696 .unwrap_err()
14697 .is::<crate::repo_lock::ResourceBlocked>()
14698 );
14699 assert!(matches!(
14700 runner.state.recovery.as_ref().unwrap().repair,
14701 crate::recovery::Repair::Started { .. }
14702 ));
14703 assert!(leftover.exists());
14704 drop(external);
14705 runner.recovery_repair(&winner).await.unwrap();
14706 assert!(!leftover.exists());
14707 assert!(matches!(
14708 runner.state.recovery.as_ref().unwrap().repair,
14709 crate::recovery::Repair::Done { .. }
14710 ));
14711 std::fs::remove_file(&lock_path).unwrap();
14713 std::fs::create_dir(&lock_path).unwrap();
14714 std::fs::write(&leftover, "untrusted edits").unwrap();
14715 let (_, _, out) = runner
14716 .ask_fixer_among(
14717 attempts,
14718 "graft",
14719 None,
14720 Some((&winner.worktree, &head)),
14721 build,
14722 )
14723 .await;
14724 assert!(matches!(out, AgentOutcome::Failed(_)));
14725 assert!(leftover.exists());
14726 assert!(!next.exists());
14727 assert_eq!(
14728 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14729 head
14730 );
14731 }
14732
14733 #[cfg(unix)]
14734 async fn registered_runner(root: &Path) -> Runner {
14735 repository_runner(root, crate::config::test_coordination()).await
14736 }
14737
14738 async fn repository_runner(
14739 root: &Path,
14740 coordination: Option<crate::config::Coordination>,
14741 ) -> Runner {
14742 ask_test_home();
14743 let repo = root.join("repo");
14744 std::fs::create_dir_all(&repo).unwrap();
14745 init_repo(&repo);
14746 if coordination.is_some() {
14747 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14748 }
14749 let mut runner = runner_at(RunStatus::Prep);
14750 runner.state.repo = repo.clone();
14751 runner.state.config.coordination = coordination;
14752 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14753 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14754 runner.state.config.graph.advise = false;
14755 runner.roles.implementers = vec![spec("implement")];
14756 runner.roles.judges = vec![spec("judge")];
14757 runner.prep().await.unwrap();
14758 runner
14759 }
14760
14761 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14762
14763 #[tokio::test]
14764 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14765 let tmp = tempfile::tempdir().unwrap();
14766 let mut runner = repository_runner(tmp.path(), None).await;
14767 let prompts = runner.state.config.prompts.clone();
14768 let run = runner.state.id.clone();
14769 let ctx = WaveCtx {
14770 carry_seats: false,
14771 pauses: None,
14772 run: &run,
14773 node: "advise",
14774 prompts: &prompts,
14775 cache: None,
14776 round: None,
14777 };
14778 let jobs: Vec<_> = ["alpha", "beta"]
14779 .into_iter()
14780 .map(|id| SeatJob {
14781 spec: scripted(id, AUTH_OUTAGE),
14782 seat: SeatState::new(id, id, 7),
14783 prompt: "propose".into(),
14784 cwd: runner.state.candidates[0].worktree.clone(),
14785 timeout: Duration::from_secs(10),
14786 allow_write: false,
14787 sessions: false,
14788 artifacts: agent::artifacts_dir(&runner.state.dir()),
14789 stem: id.into(),
14790 handover: None,
14791 })
14792 .collect();
14793 let mut losses = Vec::new();
14794 let results = ask_wave_with(
14795 jobs.clone(),
14796 runner.sem.clone(),
14797 2,
14798 &[],
14799 &ctx,
14800 &mut losses,
14801 &mut runner.state,
14802 &|_: &str| Ok(()),
14803 )
14804 .await;
14805 assert!(
14806 results
14807 .iter()
14808 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14809 );
14810 assert_eq!(runner.state.quota.len(), 2);
14811 assert!(
14812 runner
14813 .state
14814 .quota
14815 .iter()
14816 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14817 );
14818 assert!(!runner.state.advise_attempted);
14819 runner.stop_for_resource().unwrap();
14820 assert_eq!(runner.state.status, RunStatus::Blocked);
14821 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14822 assert_eq!(
14823 runner.state.resource_block.as_ref().unwrap().kind,
14824 "agent authentication"
14825 );
14826 runner.state.resource_block = None;
14829 runner.state.quota.clear();
14830 let mut mixed = jobs;
14831 mixed[1].spec.command = vec!["false".into()];
14832 let results = ask_wave_with(
14833 mixed,
14834 runner.sem.clone(),
14835 0,
14836 &[],
14837 &ctx,
14838 &mut losses,
14839 &mut runner.state,
14840 &|_: &str| Ok(()),
14841 )
14842 .await;
14843 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14844 assert!(runner.state.resource_block.is_none());
14845 assert_eq!(losses.len(), 1);
14846 }
14847
14848 #[tokio::test]
14849 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14850 let tmp = tempfile::tempdir().unwrap();
14851 let mut runner = repository_runner(tmp.path(), None).await;
14852 let implementer = scripted("implement", AUTH_OUTAGE);
14853 runner.roles.implementers = vec![implementer.clone()];
14854 runner.state.config.agents = vec![implementer];
14855 runner.implement().await.unwrap();
14856 assert_eq!(runner.state.status, RunStatus::Blocked);
14857 assert_eq!(runner.state.quota.len(), 1);
14858 assert_eq!(
14859 runner.state.quota[0].cause,
14860 agent::QuotaCause::Authentication
14861 );
14862 let candidate = &runner.state.candidates[0];
14863 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14864 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14865 }
14866
14867 #[tokio::test]
14868 #[cfg(unix)]
14869 async fn parked_implementer_persists_completed_work() {
14870 for registered in [false, true] {
14871 let tmp = tempfile::tempdir().unwrap();
14872 let mut runner = registered_runner(tmp.path()).await;
14873 if !registered {
14874 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14875 }
14876 let marker = tmp.path().join("started");
14877 let release = tmp.path().join("release");
14878 let mut implementer = spec("implement");
14879 implementer.command = vec![
14880 "sh".into(),
14881 "-c".into(),
14882 format!(
14883 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14884 marker.display(),
14885 release.display()
14886 ),
14887 ];
14888 runner.roles.implementers = vec![implementer.clone()];
14889 runner.state.config.agents = vec![implementer];
14890 let pause = runner.pause.clone();
14891 let (result, ()) = tokio::join!(runner.implement(), async {
14892 tokio::time::timeout(Duration::from_secs(10), async {
14893 while !marker.exists() {
14894 tokio::time::sleep(Duration::from_millis(10)).await;
14895 }
14896 })
14897 .await
14898 .unwrap();
14899 pause.park();
14900 std::fs::write(&release, "go").unwrap();
14901 });
14902 result.unwrap();
14903 let candidate = &runner.state.candidates[0];
14904 assert!(candidate.commits > 0, "{candidate:?}");
14905 assert!(candidate.summary.contains("saved-result"));
14906 assert_eq!(candidate.files, 1);
14907 let before = candidate.commits;
14908 runner.implement().await.unwrap();
14909 assert_eq!(runner.state.candidates[0].commits, before);
14910 }
14911 }
14912
14913 #[tokio::test]
14914 #[cfg(unix)]
14915 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14916 let tmp = tempfile::tempdir().unwrap();
14917 let mut runner = registered_runner(tmp.path()).await;
14918 let winner = runner.state.candidates[0].clone();
14919 let cache = tmp.path().join("cache");
14920 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14921 runner.state.config.graph.timeout_resource_wait = 0;
14922 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14923 let crate::cache::AcquireOutcome::Acquired(held) =
14924 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14925 else {
14926 panic!("cache must be free")
14927 };
14928 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14929 from_run: "earlier".into(),
14930 generation: 1,
14931 mode: crate::recovery::Mode::VerifyRepair,
14932 reasons: Vec::new(),
14933 reviewed_head: None,
14934 rounds: Vec::new(),
14935 open_findings: Vec::new(),
14936 evidence: None,
14937 criteria_inherited: true,
14938 repair: crate::recovery::Repair::NotStarted,
14939 });
14940 runner.recovery_repair(&winner).await.unwrap();
14941 assert!(matches!(
14942 runner.state.recovery.as_ref().unwrap().repair,
14943 crate::recovery::Repair::WaitingForRestore { .. }
14944 ));
14945 drop(held);
14946 runner.state.resource_block = None;
14947 runner.state.config.graph.timeout_resource_wait = 1;
14948 runner.recovery_repair(&winner).await.unwrap();
14949 assert!(matches!(
14950 runner.state.recovery.as_ref().unwrap().repair,
14951 crate::recovery::Repair::Done { .. }
14952 ));
14953 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14954 }
14955
14956 #[tokio::test]
14957 #[cfg(unix)]
14958 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14959 use std::os::fd::AsRawFd as _;
14960 for graft in [false, true] {
14961 let tmp = tempfile::tempdir().unwrap();
14962 let mut runner = registered_runner(tmp.path()).await;
14963 runner.state.config.graph.timeout_resource_wait = 1;
14964 let winner = runner.state.candidates[0].clone();
14965 let marker = tmp.path().join("fixer-started");
14966 let mut fixer = spec("fixer");
14967 fixer.command = vec![
14968 "sh".into(),
14969 "-c".into(),
14970 format!("echo started >> '{}'; echo done", marker.display()),
14971 ];
14972 runner.roles.fixer = Some(vec![fixer]);
14973 if graft {
14974 let mut loser = winner.clone();
14975 loser.label = 'B';
14976 loser.index = 1;
14977 runner.state.candidates.push(loser);
14978 runner.state.config.graph.graft_runner_up = true;
14979 runner.state.tally = Some(Tally {
14980 first_choice: BTreeMap::new(),
14981 borda: BTreeMap::new(),
14982 winner: winner.label,
14983 rankings: 1,
14984 unanimous_initial: true,
14985 deliberated: false,
14986 changed_votes: 0,
14987 unanimous_final: true,
14988 tie_break: None,
14989 judges: 1,
14990 present: 1,
14991 quorum: 1,
14992 met_quorum: true,
14993 uncontested: None,
14994 });
14995 runner.state.judgements.push(crate::run::Judgement {
14996 judge: 1,
14997 seat: "judge-1".into(),
14998 agent: "judge".into(),
14999 ranking: vec![winner.label, 'B'],
15000 reasons: BTreeMap::new(),
15001 confidence: None,
15002 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15003 order: vec![0, 1],
15004 failed: None,
15005 duration_ms: 0,
15006 });
15007 } else {
15008 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15009 from_run: "earlier".into(),
15010 generation: 1,
15011 mode: crate::recovery::Mode::VerifyRepair,
15012 reasons: Vec::new(),
15013 reviewed_head: None,
15014 rounds: Vec::new(),
15015 open_findings: Vec::new(),
15016 evidence: None,
15017 criteria_inherited: true,
15018 repair: crate::recovery::Repair::NotStarted,
15019 });
15020 }
15021 let reader = crate::repo_lock::SeatLease::for_checkout(
15022 &winner.worktree,
15023 false,
15024 crate::config::test_coordination().as_ref(),
15025 )
15026 .await
15027 .unwrap();
15028 let pause = runner.pause.clone();
15029 let (result, ()) = tokio::join!(
15030 async {
15031 if graft {
15032 runner.graft().await
15033 } else {
15034 runner.recovery_repair(&winner).await
15035 }
15036 },
15037 async {
15038 tokio::time::sleep(Duration::from_millis(100)).await;
15039 pause.park();
15040 }
15041 );
15042 result.unwrap();
15043 assert!(!marker.exists(), "a blocked seat must not spawn");
15044 runner.state = RunState::load(&runner.state.id).unwrap();
15046 if graft {
15047 assert_eq!(
15048 runner.state.graft.as_ref().unwrap().outcome,
15049 GraftOutcome::WaitingForRestore
15050 );
15051 } else {
15052 assert!(matches!(
15053 runner.state.recovery.as_ref().unwrap().repair,
15054 crate::recovery::Repair::WaitingForRestore { .. }
15055 ));
15056 }
15057 drop(reader);
15058 runner.pause = Pause::new();
15059 runner.state.resource_block = None;
15060 let external =
15062 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15063 assert_eq!(
15064 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15065 0
15066 );
15067 let leftover = winner.worktree.join("leftover");
15068 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15069 let result = if graft {
15070 runner.graft().await
15071 } else {
15072 runner.recovery_repair(&winner).await
15073 };
15074 assert!(
15075 result
15076 .unwrap_err()
15077 .is::<crate::repo_lock::ResourceBlocked>()
15078 );
15079 assert!(leftover.exists());
15080 assert!(!marker.exists());
15081 drop(external);
15082 runner.state.resource_block = None;
15083 let result = if graft {
15084 runner.graft().await
15085 } else {
15086 runner.recovery_repair(&winner).await
15087 };
15088 result.unwrap();
15089 assert!(!leftover.exists(), "restore must precede retry");
15090 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15091 if graft {
15092 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15093 } else {
15094 assert!(matches!(
15095 runner.state.recovery.as_ref().unwrap().repair,
15096 crate::recovery::Repair::Done { .. }
15097 ));
15098 }
15099 }
15100 }
15101
15102 #[tokio::test]
15103 #[cfg(unix)]
15104 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15105 use std::os::fd::AsRawFd as _;
15106 let tmp = tempfile::tempdir().unwrap();
15107 let mut runner = registered_runner(tmp.path()).await;
15108 runner.state.config.graph.timeout_resource_wait = 1;
15109 let winner = runner.state.candidates[0].clone();
15110 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15111 runner.state.tally = Some(Tally {
15112 first_choice: BTreeMap::new(),
15113 borda: BTreeMap::new(),
15114 winner: winner.label,
15115 rankings: 0,
15116 unanimous_initial: false,
15117 deliberated: false,
15118 changed_votes: 0,
15119 unanimous_final: false,
15120 tie_break: None,
15121 judges: 0,
15122 present: 0,
15123 quorum: 0,
15124 met_quorum: true,
15125 uncontested: Some("fixture".into()),
15126 });
15127 runner.state.graft = Some(GraftRecord {
15128 pre: pre.clone(),
15129 post: None,
15130 lines: None,
15131 limit: None,
15132 agent: None,
15133 outcome: GraftOutcome::Running,
15134 requested: Vec::new(),
15135 adoption: None,
15136 adoption_error: None,
15137 });
15138 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15139 git::commit_all(&winner.worktree, "unfinished graft")
15140 .await
15141 .unwrap();
15142 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15143 let leftover = winner.worktree.join("leftover");
15144 std::fs::write(&leftover, "untrusted edits").unwrap();
15145 let path = runner.state.repo.join(".git/updater.lock");
15146 let external = std::fs::File::create(&path).unwrap();
15147 assert_eq!(
15148 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15149 0
15150 );
15151 assert!(
15152 runner
15153 .graft()
15154 .await
15155 .unwrap_err()
15156 .is::<crate::repo_lock::ResourceBlocked>()
15157 );
15158 assert!(matches!(
15159 runner.state.graft.as_ref().unwrap().outcome,
15160 GraftOutcome::Running
15161 ));
15162 assert_eq!(
15163 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15164 unfinished
15165 );
15166 assert!(leftover.exists());
15167 drop(external);
15168 std::fs::remove_file(&path).unwrap();
15169 std::fs::create_dir(&path).unwrap();
15170 assert!(runner.graft().await.is_err());
15171 assert!(matches!(
15172 runner.state.graft.as_ref().unwrap().outcome,
15173 GraftOutcome::Running
15174 ));
15175 assert_eq!(
15176 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15177 unfinished
15178 );
15179 assert!(leftover.exists());
15180 std::fs::remove_dir(&path).unwrap();
15181 runner.graft().await.unwrap();
15182 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15183 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15184 assert!(!leftover.exists());
15185 }
15186
15187 #[tokio::test]
15188 #[cfg(unix)]
15189 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15190 use std::os::fd::AsRawFd as _;
15191 let tmp = tempfile::tempdir().unwrap();
15192 let mut runner = registered_runner(tmp.path()).await;
15193 runner.state.prep_complete = Some(false);
15194 runner.state.config.graph.timeout_resource_wait = 1;
15195 runner.state.save().unwrap();
15196 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15197 assert_eq!(
15198 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15199 0
15200 );
15201 runner.execute().await.unwrap();
15202 let disk = RunState::load(&runner.state.id).unwrap();
15203 assert!(disk.driver_exited);
15204 assert_eq!(disk.driver_pid, None);
15205 assert_eq!(runner.state.status, RunStatus::Blocked);
15206 assert_eq!(
15207 runner.state.resource_block.as_ref().unwrap().kind,
15208 "repository coordination"
15209 );
15210 assert!(runner.state.prep_failure.is_none());
15211 assert!(runner.state.seats.is_empty());
15212 runner.pause.park();
15213 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15216 .await
15217 .unwrap()
15218 .unwrap();
15219 assert!(runner.state.parked);
15220 assert!(runner.state.prep_failure.is_none());
15221 let disk = RunState::load(&runner.state.id).unwrap();
15222 assert!(disk.driver_exited);
15223 assert_eq!(disk.driver_pid, None);
15224 }
15225
15226 #[tokio::test]
15227 #[cfg(unix)]
15228 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15229 use std::os::fd::AsRawFd as _;
15230 let tmp = tempfile::tempdir().unwrap();
15231 let mut runner = registered_runner(tmp.path()).await;
15232 runner.state.prep_complete = Some(false);
15233 runner.state.config.graph.timeout_resource_wait = 30;
15234 runner.state.save().unwrap();
15235 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15236 assert_eq!(
15237 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15238 0
15239 );
15240 assert!(
15241 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15242 .await
15243 .is_err()
15244 );
15245 let disk = RunState::load(&runner.state.id).unwrap();
15246 assert!(disk.driver_exited);
15247 assert_eq!(disk.driver_pid, None);
15248 assert!(disk.status.resumable());
15249 assert!(runner.state.driver_exited);
15250 assert_eq!(runner.state.driver_pid, None);
15251 runner.state.save().unwrap();
15252 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15253 }
15254
15255 #[tokio::test]
15256 #[cfg(unix)]
15257 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15258 let tmp = tempfile::tempdir().unwrap();
15259 let mut runner = registered_runner(tmp.path()).await;
15260 let winner = runner.state.candidates[0].clone();
15261 let cache = tmp.path().join("cache");
15262 let started = tmp.path().join("pre-gate-started");
15263 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15264 runner.state.config.verify.gate =
15265 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15266 let home = crate::run::home();
15267 let owner =
15268 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15269 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15270 crate::cache::AcquireOutcome::Acquired(g) => g,
15271 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15272 };
15273 let mut gate = Box::pin(runner.run_gate(&winner));
15274 assert!(
15275 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15276 .await
15277 .is_err()
15278 );
15279 assert!(!started.exists(), "pre-gate must wait for the cache too");
15280 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15284 let editor = crate::repo_lock::SeatLease::checkout_with(
15285 &winner.worktree,
15286 true,
15287 crate::config::test_coordination().as_ref(),
15288 &wait,
15289 )
15290 .await
15291 .unwrap();
15292 drop(editor);
15293 guard.release();
15294 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15295 .await
15296 .unwrap()
15297 .unwrap();
15298 drop(gate);
15299 assert!(outcomes.iter().all(CommandOutcome::ok));
15300 assert!(started.exists());
15301 }
15302
15303 #[tokio::test]
15304 #[cfg(unix)]
15305 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15306 use std::os::fd::AsRawFd as _;
15307
15308 ask_test_home();
15309 let tmp = tempfile::tempdir().unwrap();
15310 let repo = tmp.path().join("repo");
15311 std::fs::create_dir_all(&repo).unwrap();
15312 init_repo(&repo);
15313 let common = repo.join(".git");
15314 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15315 let mut runner = runner_at(RunStatus::Prep);
15316 runner.state.repo = repo.clone();
15317 runner.state.config.coordination = crate::config::test_coordination();
15318 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15319 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15320 runner.state.config.graph.advise = false;
15321 runner.roles.implementers = vec![spec("implement")];
15322 runner.roles.judges = vec![spec("judge")];
15323 runner.prep().await.unwrap();
15324 let winner = runner.state.candidates[0].clone();
15325 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15328 let completed = tmp.path().join("seat-completed");
15329 runner.roles.implementers[0].command = vec![
15330 "sh".into(),
15331 "-c".into(),
15332 format!("touch '{}'; echo completed", completed.display()),
15333 ];
15334 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15335 assert_eq!(
15336 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15337 0
15338 );
15339 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15340 let mut implement = Box::pin(runner.implement());
15341 tokio::select! {
15342 result = &mut implement => panic!("rescue must wait: {result:?}"),
15343 result = tokio::time::timeout(Duration::from_secs(10), async {
15344 while !completed.exists() {
15345 tokio::time::sleep(Duration::from_millis(20)).await;
15346 }
15347 }) => result.unwrap(),
15348 }
15349 assert!(
15350 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15351 .await
15352 .is_err()
15353 );
15354 assert_eq!(
15355 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15356 before
15357 );
15358 assert_eq!(
15359 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15360 0
15361 );
15362 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15363 .await
15364 .unwrap()
15365 .unwrap();
15366 drop(implement);
15367 assert_ne!(
15368 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15369 before
15370 );
15371 assert!(git::is_clean(&winner.worktree).await.unwrap());
15372
15373 let started = tmp.path().join("pre-gate-started");
15374 let proceed = tmp.path().join("pre-gate-proceed");
15375 let checked = tmp.path().join("gate-ran");
15376 runner.state.config.verify.pre_gate = vec![format!(
15377 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15378 started.display(),
15379 proceed.display()
15380 )];
15381 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15382 assert_eq!(
15383 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15384 0
15385 );
15386 let mut gate = Box::pin(runner.run_gate(&winner));
15387 assert!(
15388 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15389 .await
15390 .is_err()
15391 );
15392 assert!(!started.exists());
15393 assert!(!checked.exists());
15394 assert!(git::is_clean(&winner.worktree).await.unwrap());
15395 assert_eq!(
15396 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15397 0
15398 );
15399 tokio::select! {
15400 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15401 result = tokio::time::timeout(Duration::from_secs(10), async {
15402 while !started.exists() {
15403 tokio::time::sleep(Duration::from_millis(20)).await;
15404 }
15405 }) => result.unwrap(),
15406 }
15407 assert_eq!(
15408 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15409 -1,
15410 "mechanical edits must hold the real lock"
15411 );
15412 std::fs::write(proceed, "ready").unwrap();
15413 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15414 .await
15415 .unwrap()
15416 .unwrap();
15417 drop(gate);
15418 assert!(outcomes.iter().all(CommandOutcome::ok));
15419 assert!(checked.exists());
15420 assert!(runner.state.pre_gate_commit.is_some());
15421 assert!(git::is_clean(&winner.worktree).await.unwrap());
15422 assert_eq!(
15423 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15424 0,
15425 "gate completion must release coordination"
15426 );
15427 drop(outsider);
15428 std::fs::remove_file(common.join("updater.lock")).unwrap();
15429 std::fs::create_dir(common.join("updater.lock")).unwrap();
15430 std::fs::remove_file(&started).unwrap();
15431 std::fs::remove_file(&checked).unwrap();
15432 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15433 let error = runner.run_gate(&winner).await.unwrap_err();
15434 assert!(
15435 error
15436 .to_string()
15437 .contains("acquire mandatory repository lock")
15438 );
15439 assert!(
15440 !started.exists(),
15441 "failed acquisition must prevent pre-gate"
15442 );
15443 assert!(!checked.exists(), "failed acquisition must prevent gate");
15444 assert!(git::is_clean(&winner.worktree).await.unwrap());
15445 assert_eq!(
15446 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15447 head
15448 );
15449 }
15450
15451 #[tokio::test]
15452 #[cfg(unix)]
15453 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15454 use std::os::fd::AsRawFd as _;
15455 use std::time::Duration;
15456
15457 ask_test_home();
15458 let tmp = tempfile::tempdir().unwrap();
15459 let repo = tmp.path().join("repo");
15460 std::fs::create_dir_all(&repo).unwrap();
15461 init_repo(&repo);
15462 let common = repo.join(".git");
15463 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15464 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15465 assert_eq!(
15466 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15467 0
15468 );
15469 let mut runner = runner_at(RunStatus::Prep);
15470 runner.state.repo = repo.clone();
15471 runner.state.config.coordination = crate::config::test_coordination();
15472 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15473 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15474 runner.state.config.graph.advise = true;
15475 runner.state.config.graph.advisors = 1;
15476 runner.roles.implementers = vec![spec("implement")];
15477 runner.roles.judges = vec![spec("judge")];
15478 let root = runner.state.worktree_root();
15479 let started = tmp.path().join("setup-started");
15480 let proceed = tmp.path().join("setup-proceed");
15481 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15482 run: Some(format!(
15483 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15484 started.display(),
15485 proceed.display()
15486 )),
15487 ..Default::default()
15488 }];
15489 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15490 let config = git::git(&repo, &["config", "--local", "--list"])
15491 .await
15492 .unwrap();
15493 let mut prep = Box::pin(runner.prep());
15494 assert!(
15495 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15496 .await
15497 .is_err()
15498 );
15499 assert!(
15500 !root.exists(),
15501 "contention must prevent even checkout creation"
15502 );
15503 assert!(!started.exists(), "contention must prevent setup");
15504 assert_eq!(
15505 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15506 branches
15507 );
15508 assert_eq!(
15509 git::git(&repo, &["config", "--local", "--list"])
15510 .await
15511 .unwrap(),
15512 config
15513 );
15514 drop(lock);
15515 tokio::select! {
15516 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15517 result = tokio::time::timeout(Duration::from_secs(10), async {
15518 while !started.exists() {
15519 tokio::time::sleep(Duration::from_millis(20)).await;
15520 }
15521 }) => result.unwrap(),
15522 }
15523 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15524 assert_eq!(
15525 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15526 -1
15527 );
15528 assert!(
15529 tokio::time::timeout(
15530 Duration::from_millis(100),
15531 crate::repo_lock::SeatLease::for_checkout(
15532 &repo,
15533 true,
15534 crate::config::test_coordination().as_ref()
15535 )
15536 )
15537 .await
15538 .is_err(),
15539 "preparation must also exclude local editors"
15540 );
15541 std::fs::write(proceed, "ready").unwrap();
15542 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15543 .await
15544 .unwrap()
15545 .unwrap();
15546 drop(prep);
15547 assert_eq!(runner.state.prep_complete, Some(true));
15548 assert!(root.join("judge-1").exists());
15549 assert!(root.join("advisor-1").exists());
15550 assert_eq!(
15551 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15552 0
15553 );
15554 }
15555
15556 #[tokio::test]
15557 #[cfg(unix)]
15558 async fn preparation_lock_failure_prevents_repository_changes() {
15559 ask_test_home();
15560 let tmp = tempfile::tempdir().unwrap();
15561 let repo = tmp.path().join("repo");
15562 std::fs::create_dir_all(&repo).unwrap();
15563 init_repo(&repo);
15564 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15565 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15568 let mut runner = runner_at(RunStatus::Prep);
15569 runner.state.repo = repo.clone();
15570 runner.state.config.coordination = crate::config::test_coordination();
15571 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15572 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15573 runner.roles.implementers = vec![spec("implement")];
15574 let error = runner.prep().await.unwrap_err();
15575 assert!(
15576 error
15577 .to_string()
15578 .contains("acquire mandatory repository lock")
15579 );
15580 assert!(!runner.state.worktree_root().exists());
15581 assert!(runner.state.candidates.is_empty());
15582 assert!(!repo.join(".git/config.worktree").exists());
15583 assert_eq!(
15584 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15585 "* main"
15586 );
15587 }
15588
15589 #[tokio::test]
15590 async fn preparation_records_creation_intent_before_git_can_fail() {
15591 ask_test_home();
15592 let tmp = tempfile::tempdir().unwrap();
15593 let repo = tmp.path().join("repo");
15594 std::fs::create_dir_all(&repo).unwrap();
15595 init_repo(&repo);
15596 let mut runner = runner_at(RunStatus::Prep);
15597 runner.state.repo = repo.clone();
15598 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15599 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15600 runner.state.config.blind.commit_msg_hook = false;
15601 runner.roles.implementers = vec![spec("one")];
15602 let label = blind::assign_labels(1, runner.state.seed)[0];
15603 let branch = runner.state.branch_for(label);
15604 git::git(&repo, &["branch", &branch]).await.unwrap();
15607 assert!(runner.prep().await.is_err());
15608 let saved = RunState::load(&runner.state.id).unwrap();
15609 assert!(saved.candidates.is_empty());
15610 let event = saved
15611 .events
15612 .iter()
15613 .find(|e| e.node == "prep-intent")
15614 .unwrap();
15615 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15616 assert_eq!(intent["branch"], branch);
15617 assert_eq!(intent["start"], runner.state.base_commit);
15618 assert_eq!(
15619 intent["worktree"],
15620 runner
15621 .state
15622 .worktree_root()
15623 .join(format!("cand-{label}"))
15624 .to_string_lossy()
15625 .as_ref()
15626 );
15627 assert_eq!(intent["picks"], serde_json::json!([]));
15628 }
15629
15630 #[tokio::test]
15631 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15632 ask_test_home();
15633 let tmp = tempfile::tempdir().unwrap();
15634 let repo = tmp.path().join("repo");
15635 std::fs::create_dir_all(&repo).unwrap();
15636 init_repo(&repo);
15637 let mut runner = runner_at(RunStatus::Prep);
15638 runner.state.repo = repo.clone();
15639 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15640 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15641 runner.state.sources.pick_commits.push(refs::Input {
15642 token: "missing".into(),
15643 sha: "0".repeat(40),
15644 provenance: refs::Provenance::ExplicitCli,
15645 });
15646 assert!(runner.prep().await.is_err());
15647 assert!(runner.state.candidates.is_empty());
15648 assert!(!runner.state.worktree_root().exists());
15649 }
15650
15651 #[tokio::test]
15652 async fn prose_and_legacy_inference_never_import_into_candidates() {
15653 ask_test_home();
15654 let tmp = tempfile::tempdir().unwrap();
15655 let repo = tmp.path().join("repo");
15656 std::fs::create_dir_all(&repo).unwrap();
15657 init_repo(&repo);
15658 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15659 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15660 .await
15661 .unwrap();
15662 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15663 git::git(&repo, &["add", "."]).await.unwrap();
15664 git::git(&repo, &["commit", "-m", "historical implementation"])
15665 .await
15666 .unwrap();
15667 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15668 git::git(&repo, &["checkout", "main"]).await.unwrap();
15669 for instruction in [
15670 format!("Example: magi/history/A at {source}"),
15671 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15672 format!("Never import magi/history/A or {source}"),
15673 format!("Conversation answer: I mentioned {source} only as background"),
15674 ] {
15675 let mut runner = runner_at(RunStatus::Prep);
15676 runner.state.repo = repo.clone();
15677 runner.state.base_commit = base.clone();
15678 runner.state.instruction = instruction.clone();
15679 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15680 runner.state.config.blind.commit_msg_hook = false;
15681 runner.state.seeds = vec![refs::Seed {
15682 token: "magi/history/A".into(),
15683 sha: source.clone(),
15684 kind: refs::SeedKind::Unmerged,
15685 branch: true,
15686 detail: String::new(),
15687 }];
15688 runner.roles.implementers = vec![spec("one")];
15689 assert_eq!(seeded_instruction(&runner.state), instruction);
15690 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15692 legacy.as_object_mut().unwrap().remove("sources");
15693 runner.state = serde_json::from_value(legacy).unwrap();
15694 assert_eq!(runner.state.sources, refs::Sources::default());
15695 runner.state.save().unwrap();
15696 runner.state = RunState::load(&runner.state.id).unwrap();
15697 runner.prep().await.unwrap();
15698 for candidate in &runner.state.candidates {
15699 assert_eq!(
15700 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15701 base
15702 );
15703 assert!(!candidate.worktree.join("unwanted").exists());
15704 }
15705 }
15706 }
15707
15708 #[tokio::test]
15709 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15710 ask_test_home();
15711 let tmp = tempfile::tempdir().unwrap();
15712 let repo = tmp.path().join("repo");
15713 std::fs::create_dir_all(&repo).unwrap();
15714 init_repo(&repo);
15715 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15716 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15717 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15718 git::git(&repo, &["add", "."]).await.unwrap();
15719 git::git(&repo, &["commit", "-m", "required seed"])
15720 .await
15721 .unwrap();
15722 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15723 git::git(&repo, &["checkout", "main"]).await.unwrap();
15724 let mut runner = runner_at(RunStatus::Prep);
15725 runner.state.repo = repo.clone();
15726 runner.state.base_commit = base.clone();
15727 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15728 runner.state.config.graph.advise = false;
15729 runner.state.config.blind.commit_msg_hook = false;
15730 runner.state.sources =
15731 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15732 .await
15733 .unwrap();
15734 runner.roles.implementers = vec![spec("one"), spec("two")];
15735 runner.roles.judges = vec![spec("judge")];
15736 let label = blind::assign_labels(2, runner.state.seed)[0];
15737 let branch = runner.state.branch_for(label);
15738 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15739 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15740 .await
15741 .unwrap();
15742 runner.state.candidates.push(Candidate {
15743 index: 0,
15744 label,
15745 agent: "one".into(),
15746 branch,
15747 worktree,
15748 summary: String::new(),
15749 stat: String::new(),
15750 files: 0,
15751 commits: 0,
15752 empty: false,
15753 failed: None,
15754 verified_noop: None,
15755 duration_ms: 0,
15756 folded: false,
15757 });
15758 runner.state.save().unwrap();
15759 runner.state = RunState::load(&runner.state.id).unwrap();
15761 assert_eq!(runner.state.prep_complete, Some(false));
15762 runner.prep().await.unwrap();
15763 assert_eq!(runner.state.prep_complete, Some(true));
15764 assert_eq!(runner.state.candidates.len(), 2);
15765 for candidate in &runner.state.candidates {
15766 assert_eq!(
15767 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15768 git::tree_of(&repo, &seed).await.unwrap()
15769 );
15770 assert_eq!(
15771 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15772 "required commit"
15773 );
15774 }
15775 assert!(runner.state.worktree_root().join("judge-1").exists());
15776 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15777 runner.prep().await.unwrap();
15778 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15779
15780 runner.state.prep_complete = Some(false);
15781 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15782 run: Some(format!(
15783 "test -f '{}'",
15784 tmp.path().join("dependency-ready").display()
15785 )),
15786 ..Default::default()
15787 }];
15788 assert!(runner.prep().await.is_err());
15789 assert!(runner.state.prep_failure.is_some());
15790 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15791 runner.state = RunState::load(&runner.state.id).unwrap();
15792 assert!(
15793 runner.prep().await.is_err(),
15794 "correcting the cause alone must not auto-retry"
15795 );
15796 let failed_at = runner.state.events.last().unwrap().at;
15797 assert!(
15798 runner.retry_preparation(failed_at).is_err(),
15799 "old approvals cannot unlock the hold"
15800 );
15801 let mut task = crate::queue::Task::new(
15802 "retry".into(),
15803 "retry setup".into(),
15804 repo.clone(),
15805 crate::queue::Source::Human,
15806 );
15807 task.start(runner.state.id.clone());
15808 task.hold_machine(Some("setup failed".into()));
15809 task.release_operator("cli", None);
15810 runner
15811 .retry_preparation(task.resumption.as_ref().unwrap().at)
15812 .unwrap();
15813 runner.prep().await.unwrap();
15814 assert!(runner.state.prep_failure.is_none());
15815 assert_eq!(runner.state.prep_complete, Some(true));
15816 assert_eq!(runner.state.candidates.len(), 2);
15817 assert!(runner.state.worktree_root().join("judge-1").exists());
15818 for candidate in &runner.state.candidates {
15819 assert_eq!(
15820 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15821 git::tree_of(&repo, &seed).await.unwrap()
15822 );
15823 }
15824 }
15825
15826 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15827 let mut heads = Vec::new();
15828 for candidate in candidates {
15829 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15830 }
15831 heads
15832 }
15833
15834 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15835 ask_test_home();
15836 let tmp = tempfile::tempdir().unwrap();
15837 let repo = tmp.path().join("repo");
15838 std::fs::create_dir(&repo).unwrap();
15839 init_repo(&repo);
15840 let mut runner = runner_at(RunStatus::Reviewing);
15841 runner.state.repo = repo.clone();
15842 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15843 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15844 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15845 let branch = format!("candidate-{label}");
15846 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15847 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15848 .await
15849 .unwrap();
15850 if label == 'B' {
15851 std::fs::write(worktree.join("change"), "winner").unwrap();
15852 git::rescue_commit(&worktree, "candidate change")
15853 .await
15854 .unwrap();
15855 }
15856 runner.state.candidates.push(Candidate {
15857 index,
15858 label,
15859 agent: "alpha".into(),
15860 branch,
15861 worktree,
15862 summary: "completed reply".into(),
15863 stat: String::new(),
15864 files: usize::from(label == 'B'),
15865 commits: usize::from(label == 'B'),
15866 empty: label == 'A',
15867 failed: None,
15868 verified_noop: None,
15869 duration_ms: 1,
15870 folded: false,
15871 });
15872 }
15873 runner.state.judge_skipped = true;
15874 runner.state.tally = Some(Tally {
15875 first_choice: BTreeMap::new(),
15876 borda: BTreeMap::new(),
15877 winner: 'B',
15878 rankings: 0,
15879 unanimous_initial: false,
15880 deliberated: false,
15881 changed_votes: 0,
15882 unanimous_final: false,
15883 tie_break: None,
15884 judges: 0,
15885 present: 0,
15886 quorum: 0,
15887 met_quorum: true,
15888 uncontested: Some("only B produced a change".into()),
15889 });
15890 runner.state.save().unwrap();
15891 (tmp, runner)
15892 }
15893
15894 #[tokio::test]
15895 #[cfg(unix)]
15896 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15897 for (solo, failed_checkpoint) in
15898 [(true, false), (false, false), (true, true), (false, true)]
15899 {
15900 let (_tmp, mut runner) = preservation_resume_fixture().await;
15901 if solo {
15902 runner.state.candidates.retain(|c| c.label == 'A');
15903 }
15904 runner.state.candidates[0].empty = false;
15905 runner.state.candidates[0].summary.clear();
15906 runner.state.tally = None;
15907 runner.state.judge_skipped = false;
15908 runner.state.status = RunStatus::Implementing;
15909 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15910 let mut implementer = spec("alpha");
15911 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15912 runner.roles.implementers = vec![implementer];
15913 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15914 if failed_checkpoint {
15915 std::fs::create_dir_all(&patch).unwrap();
15916 }
15917 runner.state.save().unwrap();
15918 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15919 drop(pressure);
15920 runner.state = RunState::load(&runner.state.id).unwrap();
15921 assert_eq!(runner.state.status, RunStatus::Blocked);
15922 assert!(runner.state.interrupted_implementations.contains(&'A'));
15923 assert_eq!(runner.state.candidates[0].commits, 0);
15924 assert!(!runner.state.candidates[0].empty);
15925 assert!(runner.state.candidates[0].failed.is_none());
15926 if failed_checkpoint {
15927 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15928 std::fs::remove_dir(patch).unwrap();
15929 }
15930 runner.state.resource_block = None;
15931 runner.state.config.disk.min_free_bytes = 0;
15932 runner.roles.implementers[0].command = vec![
15933 "sh".into(),
15934 "-c".into(),
15935 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15936 ];
15937 runner.implement().await.unwrap();
15938 assert!(runner.state.interrupted_implementations.is_empty());
15939 assert!(runner.state.candidate_preservation.is_empty());
15940 assert_eq!(runner.state.candidates[0].commits, 1);
15941 assert!(!runner.state.candidates[0].empty);
15942 assert!(
15943 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15944 .unwrap()
15945 .contains("retried empty interrupted seat")
15946 );
15947 if !solo {
15948 assert_eq!(runner.state.candidates[1].commits, 1);
15949 }
15950 }
15951 }
15952
15953 #[tokio::test]
15954 #[cfg(unix)]
15955 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15956 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15957 let (_tmp, mut runner) = preservation_resume_fixture().await;
15958 runner.state.candidates.retain(|c| c.label == 'A');
15959 runner.state.candidates[0].empty = false;
15960 runner.state.candidates[0].summary.clear();
15961 runner.state.tally = None;
15962 runner.state.judge_skipped = false;
15963 runner.state.status = RunStatus::Implementing;
15964 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15967 let wt = runner.state.candidates[0].worktree.clone();
15968 std::fs::write(
15969 wt.join("interrupted-source"),
15970 "retain implementation work\n",
15971 )
15972 .unwrap();
15973 if committed {
15974 git::rescue_commit(&wt, "implementation checkpoint")
15975 .await
15976 .unwrap();
15977 }
15978 let lock = git::git(
15979 &wt,
15980 &[
15981 "rev-parse",
15982 "--path-format=absolute",
15983 "--git-path",
15984 "index.lock",
15985 ],
15986 )
15987 .await
15988 .unwrap();
15989 if block_rescue {
15990 std::fs::write(&lock, "fixture-owned lock").unwrap();
15991 }
15992 let mut implementer = spec("alpha");
15993 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15994 runner.roles.implementers = vec![implementer];
15995 runner.state.save().unwrap();
15996 let result = runner.implement().await;
15997 drop(pressure);
15998 assert_eq!(result.is_err(), block_rescue);
15999 assert_eq!(
16000 runner.state.resource_block.as_ref().unwrap().kind,
16001 "disk pressure"
16002 );
16003 assert!(
16004 runner
16005 .state
16006 .events
16007 .iter()
16008 .any(|e| e.message.contains("agent execution interrupted"))
16009 );
16010 assert!(
16011 runner
16012 .state
16013 .dir()
16014 .join("artifacts/candidate-A-recovery/staged.patch")
16015 .exists()
16016 );
16017 if block_rescue {
16018 let saved = RunState::load(&runner.state.id).unwrap();
16019 assert!(saved.candidate_preservation.contains_key(&'A'));
16020 assert!(!saved.candidates[0].empty);
16021 assert_eq!(
16022 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16023 "retain implementation work\n"
16024 );
16025 std::fs::remove_file(lock).unwrap();
16026 runner.state = saved;
16027 } else {
16028 assert_eq!(runner.state.status, RunStatus::Blocked);
16029 assert_eq!(runner.state.candidates[0].commits, 1);
16030 }
16031 runner.roles.implementers.clear();
16034 runner.state.config.disk.min_free_bytes = 0;
16035 runner.state.resource_block = None;
16036 runner.implement().await.unwrap();
16037 assert!(runner.state.candidate_preservation.is_empty());
16038 assert_eq!(runner.state.candidates[0].commits, 1);
16039 assert!(!runner.state.candidates[0].empty);
16040 assert!(git::is_clean(&wt).await.unwrap());
16041 assert!(
16042 crate::run::read_artifact(&runner.state, "cand-A.patch")
16043 .unwrap()
16044 .contains("retain implementation work")
16045 );
16046 }
16047 }
16048
16049 #[tokio::test]
16050 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16051 let (_tmp, mut runner) = preservation_resume_fixture().await;
16052 runner.implement().await.unwrap();
16054 assert!(runner.state.candidate_preservation.is_empty());
16055 assert!(
16056 !runner
16057 .state
16058 .dir()
16059 .join("artifacts/candidate-A-recovery")
16060 .exists()
16061 );
16062 runner.fold_losers().await.unwrap();
16063 assert!(!runner.state.candidates[0].worktree.exists());
16064 for schema in [14, crate::run::SCHEMA] {
16065 runner.state.schema = schema;
16066 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16067 runner.state.save().unwrap();
16068 runner.state = RunState::load(&runner.state.id).unwrap();
16069 runner.implement().await.unwrap();
16070 assert!(runner.state.candidate_preservation.is_empty());
16071 assert!(runner.state.candidates[0].folded);
16072 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16073 }
16074 }
16075
16076 #[tokio::test]
16077 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16078 let (_tmp, mut runner) = preservation_resume_fixture().await;
16079 runner.state.candidates.retain(|c| c.label == 'A');
16080 runner.state.tally = None;
16081 runner.state.judge_skipped = false;
16082 let candidate = &mut runner.state.candidates[0];
16083 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16084 candidate.failed = Some("deployment validation unavailable".into());
16085 std::fs::write(
16086 candidate.worktree.join("repair"),
16087 "future launcher policy\n",
16088 )
16089 .unwrap();
16090 runner
16091 .state
16092 .candidate_preservation
16093 .insert('A', "awaiting persistence".into());
16094 runner.persist_candidates().await.unwrap();
16095 let candidate = &runner.state.candidates[0];
16096 assert!(!candidate.empty);
16097 assert_eq!(candidate.commits, 1);
16098 assert_eq!(candidate.files, 1);
16099 assert!(candidate.failed.is_none());
16100 assert!(candidate.summary.contains("EPERM"));
16101 assert!(
16102 crate::run::read_artifact(&runner.state, "cand-A.patch")
16103 .unwrap()
16104 .contains("future launcher policy")
16105 );
16106 assert!(runner.state.candidate_preservation.is_empty());
16107 }
16108
16109 #[tokio::test]
16110 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16111 let (_tmp, mut runner) = preservation_resume_fixture().await;
16112 let wt = runner.state.candidates[0].worktree.clone();
16113 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16114 runner.state.schema = 14;
16115 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16116 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16117 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16118 for _ in 0..2 {
16119 assert!(runner.implement().await.is_err());
16120 assert_eq!(runner.state.status, RunStatus::Blocked);
16121 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16122 assert!(
16123 runner.state.candidates[0].empty,
16124 "old verdict must remain unusable"
16125 );
16126 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16127 assert!(runner.state.judge_skipped);
16128 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16129 assert!(
16130 crate::run::read_artifact(&runner.state, "cand-A.patch")
16131 .unwrap()
16132 .contains("owned edits")
16133 );
16134 assert!(wt.exists());
16135 assert_eq!(
16136 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16137 .await
16138 .unwrap(),
16139 1
16140 );
16141 runner.state = RunState::load(&runner.state.id).unwrap();
16142 }
16143 }
16144
16145 #[tokio::test]
16146 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16147 let (_tmp, mut runner) = preservation_resume_fixture().await;
16148 let wt = runner.state.candidates[0].worktree.clone();
16149 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16150 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16151 let index_lock = PathBuf::from(
16152 git::git(
16153 &wt,
16154 &[
16155 "rev-parse",
16156 "--path-format=absolute",
16157 "--git-path",
16158 "index.lock",
16159 ],
16160 )
16161 .await
16162 .unwrap(),
16163 );
16164 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16165 runner.state.schema = 14;
16166 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16167 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16168 assert!(runner.implement().await.is_err());
16169 assert!(runner.after_implement().is_err());
16170 assert!(runner.fold_losers().await.is_err());
16171 assert_eq!(
16172 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16173 "failed seat edits"
16174 );
16175 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16176 runner.state = RunState::load(&runner.state.id).unwrap();
16177 std::fs::remove_file(index_lock).unwrap();
16178 assert!(runner.implement().await.is_err());
16180 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16181 assert!(runner.state.candidates[0].empty);
16182 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16183 assert!(
16184 crate::run::read_artifact(&runner.state, "cand-A.patch")
16185 .unwrap()
16186 .contains("failed seat edits")
16187 );
16188 assert!(wt.exists());
16189 }
16190
16191 #[tokio::test]
16192 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16193 let (_tmp, mut runner) = preservation_resume_fixture().await;
16194 let repo = runner.state.repo.clone();
16195 let common = git::git(
16196 &repo,
16197 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16198 )
16199 .await
16200 .unwrap();
16201 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16202 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16203 assert!(runner.fold_losers().await.is_err());
16204 assert!(!runner.state.candidates[0].worktree.exists());
16205 assert!(!runner.state.candidates[0].folded);
16206 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16207 runner.state = RunState::load(&runner.state.id).unwrap();
16208 std::fs::remove_file(branch_lock).unwrap();
16209 let judge = runner.state.worktree_root().join("judge-1");
16210 git::worktree_add_detached(&repo, &judge, "main")
16211 .await
16212 .unwrap();
16213 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16214 runner.roles.judges = vec![spec("judge")];
16215 assert!(runner.fold_losers().await.is_err());
16216 runner.state = RunState::load(&runner.state.id).unwrap();
16217 assert!(
16218 runner.state.candidates[0].folded,
16219 "candidate progress saved before judge failure"
16220 );
16221 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16222 assert!(judge.join("uncommitted").exists());
16223 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16224 runner.fold_losers().await.unwrap();
16225 assert!(!judge.exists());
16226 runner.fold_losers().await.unwrap();
16227 }
16228
16229 #[tokio::test]
16230 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16231 let (_tmp, mut runner) = preservation_resume_fixture().await;
16232 runner.state.candidates.retain(|c| c.label == 'B');
16233 runner.state.tally = None;
16234 runner.state.judge_skipped = false;
16235 runner.state.prep_complete = Some(true);
16236 runner.state.config.graph.advise = false;
16237 runner.state.config.graph.selected_implementer = Some("alpha".into());
16238 let wt = runner.state.candidates[0].worktree.clone();
16239 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16240 runner
16241 .state
16242 .candidate_preservation
16243 .insert('B', "awaiting persistence".into());
16244 let index_lock = git::git(
16245 &wt,
16246 &[
16247 "rev-parse",
16248 "--path-format=absolute",
16249 "--git-path",
16250 "index.lock",
16251 ],
16252 )
16253 .await
16254 .unwrap();
16255 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16256 runner.state.save().unwrap();
16257 assert!(runner.execute_graph().await.is_err());
16258 assert_eq!(runner.state.status, RunStatus::Blocked);
16259 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16260 std::fs::remove_file(index_lock).unwrap();
16261 for _ in 0..2 {
16264 runner.state = RunState::load(&runner.state.id).unwrap();
16265 runner.pause.park();
16266 runner.execute_graph().await.unwrap();
16267 assert_eq!(runner.state.status, RunStatus::Implementing);
16268 assert!(runner.state.candidate_preservation.is_empty());
16269 assert_eq!(runner.state.candidates[0].commits, 2);
16270 assert!(runner.state.tally.is_none());
16271 }
16272 runner.implement().await.unwrap();
16273 assert_eq!(runner.state.status, RunStatus::Judging);
16274 runner.state.status = RunStatus::Blocked;
16276 runner.state.candidates[0].failed = Some("implementation failed".into());
16277 runner.state.save().unwrap();
16278 let error = runner.execute_graph().await.unwrap_err();
16279 assert!(
16280 error
16281 .to_string()
16282 .contains("selected implementation is held")
16283 );
16284 }
16285
16286 #[tokio::test]
16287 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16288 let (_tmp, mut runner) = preservation_resume_fixture().await;
16289 let repo = runner.state.repo.clone();
16290 let candidate = runner.state.candidates[0].clone();
16291 let registered = git::worktree_holding(&repo, &candidate.branch)
16294 .await
16295 .unwrap()
16296 .unwrap();
16297 assert_eq!(
16298 registered.canonicalize().unwrap(),
16299 candidate.worktree.canonicalize().unwrap()
16300 );
16301 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16302 assert_eq!(
16303 git::worktree_holding(&repo, &candidate.branch)
16304 .await
16305 .unwrap(),
16306 Some(registered)
16307 );
16308 assert!(
16309 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16310 "the stale registration initially keeps the branch checked out"
16311 );
16312 runner.fold_losers().await.unwrap();
16313 assert!(runner.state.candidates[0].folded);
16314 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16315 assert_eq!(
16316 git::worktree_holding(&repo, &candidate.branch)
16317 .await
16318 .unwrap(),
16319 None
16320 );
16321 runner.state = RunState::load(&runner.state.id).unwrap();
16322 runner.fold_losers().await.unwrap();
16323 }
16324
16325 #[tokio::test]
16326 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16327 let (_tmp, mut runner) = preservation_resume_fixture().await;
16328 let repo = runner.state.repo.clone();
16329 let candidate = runner.state.candidates[0].clone();
16330 let winner = runner.state.candidates[1].clone();
16331 let registered = git::worktree_holding(&repo, &candidate.branch)
16332 .await
16333 .unwrap()
16334 .unwrap();
16335 assert_eq!(
16336 registered.canonicalize().unwrap(),
16337 candidate.worktree.canonicalize().unwrap()
16338 );
16339 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16340 assert_eq!(
16341 git::worktree_holding(&repo, &candidate.branch)
16342 .await
16343 .unwrap(),
16344 Some(registered)
16345 );
16346 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16347
16348 for _ in 0..2 {
16349 fold_run(&mut runner.state, false, &crate::run::home())
16350 .await
16351 .unwrap();
16352 runner.state = RunState::load(&runner.state.id).unwrap();
16353 assert!(runner.state.candidates[0].folded);
16354 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16355 assert_eq!(
16356 git::worktree_holding(&repo, &candidate.branch)
16357 .await
16358 .unwrap(),
16359 None
16360 );
16361 assert!(!runner.state.candidates[1].folded);
16362 assert!(winner.worktree.exists());
16363 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16364 }
16365 }
16366
16367 #[tokio::test]
16368 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16369 ask_test_home();
16370 let tmp = tempfile::tempdir().unwrap();
16371 let repo = tmp.path().join("repo");
16372 std::fs::create_dir(&repo).unwrap();
16373 init_repo(&repo);
16374 let mut runner = runner_at(RunStatus::Implementing);
16375 runner.state.repo = repo.clone();
16376 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16377 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16378 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16379 let branch = format!("candidate-{label}");
16380 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16381 git::worktree_add_branch(&repo, &wt, &branch, "main")
16382 .await
16383 .unwrap();
16384 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16385 git::git(&wt, &["add", "README.md"]).await.unwrap();
16386 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16387 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16388 runner.state.candidates.push(Candidate {
16389 index,
16390 label,
16391 agent: "alpha".to_owned(),
16392 branch,
16393 worktree: wt,
16394 summary: "completed implementation".to_owned(),
16395 stat: String::new(),
16396 files: 0,
16397 commits: 0,
16398 empty: false,
16399 failed: None,
16400 verified_noop: None,
16401 duration_ms: 42,
16402 folded: false,
16403 });
16404 runner
16405 .state
16406 .candidate_preservation
16407 .insert(label, "awaiting persistence".to_owned());
16408 }
16409 let wt = runner.state.candidates[0].worktree.clone();
16410 let index_lock = git::git(
16411 &wt,
16412 &[
16413 "rev-parse",
16414 "--path-format=absolute",
16415 "--git-path",
16416 "index.lock",
16417 ],
16418 )
16419 .await
16420 .unwrap();
16421 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16422 runner.state.save().unwrap();
16423 assert!(runner.persist_candidates().await.is_err());
16424 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16425 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16426 assert_eq!(
16427 runner.state.candidates[1].commits, 1,
16428 "other results are still collected"
16429 );
16430 assert!(
16431 !runner.state.candidates[0].empty,
16432 "failed rescue is not no change"
16433 );
16434 assert_eq!(
16435 runner.state.candidates[0].summary,
16436 "completed implementation"
16437 );
16438 assert!(runner.after_implement().is_err());
16439 let home = tmp.path().join("fold-home");
16440 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16441 assert!(wt.join("new-file").exists());
16442 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16443 assert!(recovery.join("staged.patch").exists());
16444 assert!(recovery.join("unstaged.patch").exists());
16445 assert_eq!(
16446 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16447 "untracked edits\n"
16448 );
16449 runner.state.begin_pass();
16450 runner.state.end_pass_parked(ParkCause::Upgrade {
16451 from: "1.0".into(),
16452 to: Some("1.1".into()),
16453 });
16454 runner.state.save().unwrap();
16455 runner.state = RunState::load(&runner.state.id).unwrap();
16456 assert!(matches!(
16457 runner.state.passes.last().unwrap().end,
16458 Some(crate::run::PassEnd::Parked {
16459 cause: ParkCause::Upgrade { .. }
16460 })
16461 ));
16462 std::fs::remove_file(index_lock).unwrap();
16463 runner.implement().await.unwrap();
16465 assert!(runner.state.candidate_preservation.is_empty());
16466 assert!(
16467 runner
16468 .state
16469 .candidates
16470 .iter()
16471 .all(|c| c.commits == 1 && !c.empty)
16472 );
16473 assert!(git::is_clean(&wt).await.unwrap());
16474 assert!(
16475 crate::run::read_artifact(&runner.state, "cand-A.patch")
16476 .unwrap()
16477 .contains("later tracked edits")
16478 );
16479 }
16480
16481 #[tokio::test]
16482 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16483 ask_test_home();
16484 let tmp = tempfile::tempdir().unwrap();
16485 let repo = tmp.path().join("repo");
16486 std::fs::create_dir(&repo).unwrap();
16487 init_repo(&repo);
16488 let wt = tmp.path().join("candidate");
16489 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16490 .await
16491 .unwrap();
16492 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16493 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16494 assert!(wt.join("uncommitted").exists());
16495 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16496 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16497 let mut runner = runner_at(RunStatus::Implementing);
16498 runner.state.repo = repo.clone();
16499 runner.state.base_commit = "missing-base".to_owned();
16500 runner.state.candidates.push(Candidate {
16501 index: 0,
16502 label: 'A',
16503 agent: "alpha".into(),
16504 branch: "candidate".into(),
16505 worktree: wt.clone(),
16506 summary: "completed".into(),
16507 stat: String::new(),
16508 files: 0,
16509 commits: 0,
16510 empty: false,
16511 failed: None,
16512 verified_noop: None,
16513 duration_ms: 1,
16514 folded: false,
16515 });
16516 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16517 assert!(
16518 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16519 .await
16520 .is_err()
16521 );
16522 assert!(!runner.state.candidates[0].folded);
16523 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16524 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16525 runner
16526 .state
16527 .candidate_preservation
16528 .insert('A', "pending".into());
16529 assert!(runner.persist_candidates().await.is_err());
16530 assert!(!runner.state.candidates[0].empty);
16531 assert!(!runner.state.candidates[0].folded);
16532 assert!(wt.exists());
16533 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16534 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16535 std::fs::create_dir_all(&patch_path).unwrap();
16536 assert!(
16537 runner.persist_candidates().await.is_err(),
16538 "artifact failure stays pending"
16539 );
16540 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16541 assert!(!runner.state.candidates[0].empty);
16542 std::fs::remove_dir(patch_path).unwrap();
16543 runner.persist_candidates().await.unwrap();
16544 assert!(
16545 runner.state.candidates[0].empty,
16546 "only successful inspection confirms no change"
16547 );
16548 }
16549
16550 fn init_repo(dir: &Path) {
16551 let run = |args: &[&str]| {
16552 let out = std::process::Command::new("git")
16553 .args(args)
16554 .current_dir(dir)
16555 .quiet()
16556 .output()
16557 .expect("spawn git");
16558 assert!(
16559 out.status.success(),
16560 "git {args:?} failed: {}",
16561 String::from_utf8_lossy(&out.stderr)
16562 );
16563 };
16564 run(&["init", "-b", "main"]);
16565 run(&["config", "user.name", "magi test"]);
16566 run(&["config", "user.email", "magi@example.com"]);
16567 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16568 run(&["add", "-A"]);
16569 run(&["commit", "-m", "init"]);
16570 }
16571
16572 fn ask_test_home() {
16580 crate::run::pin_test_home();
16581 }
16582
16583 #[test]
16584 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16585 ask_test_home();
16586 let mut runner = runner_at(RunStatus::Prep);
16587 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16588 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16591 let error = guard
16592 .finish(
16593 &mut runner.state,
16594 Err(
16595 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16596 .into(),
16597 ),
16598 )
16599 .unwrap_err();
16600 assert_eq!(
16601 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16602 std::io::ErrorKind::PermissionDenied
16603 );
16604 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16605 let message = format!("{error:#}");
16606 assert!(message.contains("original refusal"));
16607 assert!(!message.contains("could not save terminal"));
16608 assert_eq!(
16609 error.chain().count(),
16610 2,
16611 "only startup attribution wraps the original error"
16612 );
16613 let saved = RunState::load(&runner.state.id).unwrap();
16614 assert_eq!(saved.status, RunStatus::Blocked);
16615 assert!(!saved.startup.unwrap().complete);
16616 }
16617
16618 #[test]
16619 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16620 ask_test_home();
16621 let mut runner = runner_at(RunStatus::Prep);
16622 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16623 let id = runner.state.id.clone();
16624 drop(guard);
16625 let saved = RunState::load(&id).unwrap();
16626 assert_eq!(saved.status, RunStatus::Blocked);
16627 assert!(!saved.startup.as_ref().unwrap().complete);
16628 assert!(
16629 saved
16630 .startup
16631 .unwrap()
16632 .failure
16633 .unwrap()
16634 .contains("interrupted")
16635 );
16636 assert!(Runner::resume(&id).is_err());
16637 }
16638
16639 #[test]
16640 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16641 ask_test_home();
16642 let mut runner = runner_at(RunStatus::Prep);
16643 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16644 let error = guard
16645 .finish(
16646 &mut runner.state,
16647 Err(
16648 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16649 .into(),
16650 ),
16651 )
16652 .unwrap_err();
16653 assert_eq!(
16654 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16655 std::io::ErrorKind::PermissionDenied
16656 );
16657 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16658 let mut task = crate::queue::Task::new(
16659 "review".into(),
16660 "criteria".into(),
16661 runner.state.repo.clone(),
16662 crate::queue::Source::Human,
16663 );
16664 task.record_startup_failure(failure.clone());
16665 task.record_startup_failure(failure.clone());
16666 assert_eq!(task.startup_failures.len(), 1);
16667 assert!(task.runs.is_empty());
16668 assert_eq!(task.attempts, 0);
16669 }
16670
16671 #[cfg(unix)]
16672 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16673 ask_test_home();
16674 let tmp = tempfile::tempdir().unwrap();
16675 let repo = tmp.path().join("repo");
16676 std::fs::create_dir(&repo).unwrap();
16677 init_repo(&repo);
16678 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16679 let origin = tmp.path().join("origin.git");
16680 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16681 .await
16682 .unwrap();
16683 git::git(
16684 &repo,
16685 &["remote", "add", "origin", origin.to_str().unwrap()],
16686 )
16687 .await
16688 .unwrap();
16689 git::git(&repo, &["checkout", "-b", "fix/retained"])
16690 .await
16691 .unwrap();
16692 std::fs::write(repo.join("retained"), "retained change").unwrap();
16693 git::git(&repo, &["add", "."]).await.unwrap();
16694 git::git(&repo, &["commit", "-m", "retained implementation"])
16695 .await
16696 .unwrap();
16697 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16698 git::git(&repo, &["checkout", "main"]).await.unwrap();
16699 std::fs::write(repo.join("base-change"), "new base").unwrap();
16700 git::git(&repo, &["add", "."]).await.unwrap();
16701 git::git(&repo, &["commit", "-m", "base moved"])
16702 .await
16703 .unwrap();
16704 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16705 git::git(&repo, &["checkout", "fix/retained"])
16706 .await
16707 .unwrap();
16708 let mut runner = runner_at(RunStatus::Voting);
16709 runner.state.repo = repo.clone();
16710 runner.state.base_commit = base;
16711 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16712 runner.state.config.disk.min_free_bytes = 0;
16713 runner.state.config.graph.review_rounds = 1;
16714 runner.state.config.verify.e2e = vec!["true".into()];
16715 runner.state.config.verify.gate = vec!["true".into()];
16716 runner.state.candidates = vec![Candidate {
16717 index: 0,
16718 label: 'A',
16719 agent: "fixture".into(),
16720 branch: "fix/retained".into(),
16721 worktree: repo.clone(),
16722 summary: String::new(),
16723 stat: String::new(),
16724 files: 1,
16725 commits: 1,
16726 empty: false,
16727 failed: None,
16728 verified_noop: None,
16729 duration_ms: 0,
16730 folded: false,
16731 }];
16732 runner.state.tally = Some(Tally {
16733 first_choice: BTreeMap::new(),
16734 borda: BTreeMap::new(),
16735 winner: 'A',
16736 rankings: 0,
16737 unanimous_initial: true,
16738 deliberated: false,
16739 changed_votes: 0,
16740 unanimous_final: true,
16741 tie_break: None,
16742 judges: 0,
16743 present: 0,
16744 quorum: 0,
16745 met_quorum: true,
16746 uncontested: Some("review-only".into()),
16747 });
16748 let mut task = crate::queue::Task::new(
16749 "fixture".into(),
16750 "fixture".into(),
16751 repo,
16752 crate::queue::Source::Human,
16753 );
16754 task.link_run(&runner.state.id);
16755 crate::queue::Queue::open().put(&mut task).unwrap();
16756 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16757 runner.state.base_sync = Some(BaseSync {
16758 tip: String::new(),
16759 behind: 1,
16760 attempts: BASE_SYNC_ROUNDS,
16761 conflict: None,
16762 already_in: None,
16763 });
16764 runner.sync_to_base().await.unwrap();
16765 assert_eq!(runner.state.status, RunStatus::Blocked);
16766 (tmp, runner, head, task.id)
16767 }
16768
16769 #[cfg(unix)]
16770 #[tokio::test]
16771 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16772 for routed in [false, true] {
16773 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16774 let home = crate::run::home();
16775 let queue = crate::queue::Queue::open();
16776 let task = queue.get(&owner).unwrap();
16777 runner.state.task_instruction = Some(task.instruction.clone());
16778 runner.state.driver_exited = true;
16779 runner.state.save().unwrap();
16780 let questions = crate::ask::Questions::at(home.join("questions"));
16781 let mut q = crate::ask::Question::new(
16782 runner.state.id.clone(),
16783 "base-recovery".into(),
16784 "owner".into(),
16785 "Recover bounded base movement?".into(),
16786 String::new(),
16787 vec!["recover".into()],
16788 );
16789 q.actions.insert(
16790 "recover".into(),
16791 crate::ask::ChoiceAction::RecoverBase {
16792 run: runner.state.id.clone(),
16793 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16794 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16795 },
16796 );
16797 q.routed_task = Some(owner.clone());
16798 q.answer(crate::ask::Answer::Choice("recover".into()))
16799 .unwrap();
16800 questions.put(&mut q).unwrap();
16801 crate::base_recovery_control::recover(
16802 &queue,
16803 &home,
16804 crate::base_recovery_control::Request {
16805 run: runner.state.id.clone(),
16806 head: head.clone(),
16807 actor: "decisions".into(),
16808 request: q.id.clone(),
16809 decision: Some((owner.clone(), q.answered_at.unwrap())),
16810 expected_stop: Some((
16811 runner.state.base_sync_recovery.stop.clone().unwrap(),
16812 runner.state.base_sync_recovery.episodes.last().unwrap().id,
16813 )),
16814 },
16815 )
16816 .await
16817 .unwrap();
16818 if routed {
16819 questions
16820 .update(&q.id, |q| {
16821 q.routed_task = Some("another-owner".into());
16822 Ok(())
16823 })
16824 .unwrap();
16825 } else {
16826 let mut task = queue.get(&owner).unwrap();
16827 task.hold_manual(Some("new owner decision".into()));
16828 task.status = crate::queue::TaskStatus::Queued;
16830 queue.put(&mut task).unwrap();
16831 }
16832 runner.state = RunState::load(&runner.state.id).unwrap();
16833 runner.sync_to_base().await.unwrap();
16834 let approval = runner
16835 .state
16836 .base_sync_recovery
16837 .authorizations
16838 .last()
16839 .unwrap();
16840 assert!(approval.consumed);
16841 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16842 "routing"
16843 } else {
16844 "owner decision"
16845 }));
16846 assert_eq!(
16847 runner.state.base_sync.as_ref().unwrap().attempts,
16848 BASE_SYNC_ROUNDS
16849 );
16850 assert_eq!(
16851 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16852 head
16853 );
16854 }
16855 }
16856
16857 #[cfg(unix)]
16858 #[tokio::test]
16859 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16860 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16861 if std::env::var_os(CHILD).is_none() {
16862 let home = tempfile::tempdir().unwrap();
16863 let output = std::process::Command::new(std::env::current_exe().unwrap())
16864 .args([
16865 "--exact",
16866 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16867 "--nocapture",
16868 ])
16869 .env(CHILD, "1")
16870 .env("MAGI_HOME", home.path())
16871 .env("MAGI_WORKER_HOME", home.path())
16872 .env("MAGI_WORKER_ID", "abcdef")
16873 .output()
16874 .unwrap();
16875 assert!(
16876 output.status.success()
16877 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16878 "{}\n{}",
16879 String::from_utf8_lossy(&output.stdout),
16880 String::from_utf8_lossy(&output.stderr)
16881 );
16882 return;
16883 }
16884 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16885 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16886 let home = crate::run::home();
16887 let queue = crate::queue::Queue::open();
16888 let mut owner = queue.get(&task).unwrap();
16889 owner.release_operator("test-release", None);
16890 owner.start(runner.state.id.clone());
16891 queue.put(&mut owner).unwrap();
16892 runner.state.driver_exited = true;
16893 runner.state.task_instruction = Some(owner.instruction.clone());
16894 runner.state.save().unwrap();
16895 let _claim = queue.claim(&task).unwrap();
16896 let reserve = |id: &str| {
16897 let directory = home.join("workers/jobs").join(id);
16898 std::fs::create_dir_all(&directory).unwrap();
16899 let job = crate::worker::Job {
16900 id: id.into(),
16901 sequence: 1,
16902 kind: "task".into(),
16903 key: format!("task:{task}"),
16904 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16905 generation: "fixture-generation".into(),
16906 version: None,
16907 status: crate::worker::Status::Running,
16908 result: None,
16909 pid: Some(std::process::id()),
16910 created_at: 0,
16911 finished_at: None,
16912 };
16913 std::fs::write(
16914 directory.join("job.json"),
16915 serde_json::to_vec(&job).unwrap(),
16916 )
16917 .unwrap();
16918 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16919 };
16920 let _own = reserve("abcdef");
16921 let other = reserve("fedcba");
16922 assert!(
16923 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16924 .await
16925 .is_err()
16926 );
16927 assert!(
16928 RunState::load(&runner.state.id)
16929 .unwrap()
16930 .base_sync_recovery
16931 .authorizations
16932 .is_empty()
16933 );
16934 drop(other);
16935 assert!(crate::daemon::is_working_on(
16936 &home,
16937 &runner.state.id,
16938 Timestamp::now()
16939 ));
16940 assert!(
16941 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16942 );
16943 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16944 .await
16945 .unwrap();
16946 assert!(
16947 RunState::load(&runner.state.id)
16948 .unwrap()
16949 .base_recovery_pending()
16950 );
16951 }
16952
16953 #[cfg(unix)]
16954 #[tokio::test]
16955 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16956 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16957 let home = crate::run::home();
16958 runner.state.driver_pid = Some(std::process::id());
16959 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16960 runner.state.driver_exited = false;
16961 runner.state.begin_pass();
16962 runner.state.save().unwrap();
16963 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16964 driver.record(&runner.state).unwrap();
16965 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16966 drop(driver);
16968 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16969 assert_eq!(
16970 runner.state.liveness_under(&home, false),
16971 crate::run::Liveness::Dead
16972 );
16973 let queue = crate::queue::Queue::open();
16974 let _claim = queue.claim(&task).unwrap();
16975 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16976 .await
16977 .unwrap();
16978 let disk = RunState::load(&runner.state.id).unwrap();
16979 assert!(!disk.driver_exited);
16980 assert!(disk.base_recovery_pending());
16981 }
16982
16983 #[cfg(unix)]
16984 #[tokio::test]
16985 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16986 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16987 if std::env::var_os(CHILD).is_none() {
16988 use std::os::unix::fs::PermissionsExt;
16989 let tmp = tempfile::tempdir().unwrap();
16990 let gh = tmp.path().join("gh");
16991 std::fs::write(&gh, r#"#!/bin/sh
16992case "$1 $2" in
16993 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16994 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16995 *) exit 2 ;;
16996esac
16997"#).unwrap();
16998 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16999 let mut paths = vec![tmp.path().to_path_buf()];
17000 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17001 let output = std::process::Command::new(std::env::current_exe().unwrap())
17002 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17003 .env(CHILD, "1")
17004 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17005 .env("PATH", std::env::join_paths(paths).unwrap())
17006 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17007 .output().unwrap();
17008 assert!(
17009 output.status.success()
17010 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17011 "{}\n{}",
17012 String::from_utf8_lossy(&output.stdout),
17013 String::from_utf8_lossy(&output.stderr)
17014 );
17015 return;
17016 }
17017 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17018 let repo = runner.state.repo.clone();
17019 git::git(&repo, &["checkout", "main"]).await.unwrap();
17020 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17021 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17022 git::git(&repo, &["checkout", "fix/retained"])
17023 .await
17024 .unwrap();
17025 runner.state.pr = Some(crate::run::PrRecord {
17026 url: "https://example.test/project/pull/1".into(),
17027 number: 1,
17028 state: "open".into(),
17029 checks: "unknown".into(),
17030 round: 0,
17031 rounds: 1,
17032 red_at_merge: Vec::new(),
17033 });
17034 runner.sync_to_base().await.unwrap();
17035 assert_eq!(runner.state.status, RunStatus::Blocked);
17036 assert_eq!(
17037 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17038 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17039 );
17040 assert!(runner.state.base_sync_closure_pending());
17041 assert!(crate::handover::can_resume(&runner.state));
17042 let queue = crate::queue::Queue::open();
17043 assert!(!crate::base_sync_recovery::held_task(
17044 &queue,
17045 &queue.get(&task).unwrap()
17046 ));
17047 assert!(runner.state.authorize_base_sync(&task).is_err());
17048 runner.state.base_sync_recovery.stop = None;
17050 runner.state.status = RunStatus::Voting;
17051 runner.state.gate_ran = true;
17052 runner.state.config.graph.review_rounds = 0;
17053 runner.state.save().unwrap();
17054 assert!(crate::handover::can_resume(&runner.state));
17055 runner.state = RunState::load(&runner.state.id).unwrap();
17056 runner.execute_graph().await.unwrap();
17057 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17058 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17059 assert_eq!(
17060 runner.state.base_sync.as_ref().unwrap().attempts,
17061 BASE_SYNC_ROUNDS
17062 );
17063 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17064 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17065 assert!(runner.state.reviews.is_empty());
17066 assert!(runner.state.gate.is_empty());
17067 }
17068
17069 #[cfg(unix)]
17070 #[tokio::test]
17071 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17072 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17073 runner.state.driver_exited = true;
17074 runner.state.save().unwrap();
17075 let home = crate::run::home();
17076 let queue = crate::queue::Queue::open();
17077 let _claim = queue.claim(&task).unwrap();
17078 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17079 assert!(
17080 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17081 .await
17082 .is_err()
17083 );
17084 drop(lease);
17085 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17086 assert!(
17087 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17088 .await
17089 .is_err()
17090 );
17091 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17092 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17093 .await
17094 .unwrap();
17095 let disk = RunState::load(&runner.state.id).unwrap();
17096 assert!(disk.base_recovery_pending());
17097 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17098 assert!(
17099 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17100 .await
17101 .is_err()
17102 );
17103 assert_eq!(
17104 RunState::load(&runner.state.id)
17105 .unwrap()
17106 .base_sync_recovery
17107 .authorizations
17108 .len(),
17109 1
17110 );
17111 let mut owner = queue.get(&task).unwrap();
17112 owner.runs.push("20260101-000000-ffff".into());
17113 queue.put(&mut owner).unwrap();
17114 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17115 .await
17116 .unwrap_err();
17117 assert!(error.to_string().contains("run task ownership changed"));
17118 }
17119
17120 #[cfg(unix)]
17121 #[tokio::test]
17122 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17123 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17124 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17125 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17126 legacy["status"] = serde_json::json!("voting");
17127 runner.state = serde_json::from_value(legacy).unwrap();
17128 runner.state.save().unwrap();
17129 for _ in 0..3 {
17130 runner.execute_graph().await.unwrap();
17131 assert_eq!(runner.state.status, RunStatus::Blocked);
17132 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17133 assert!(!crate::handover::can_resume(&runner.state));
17134 assert_eq!(
17135 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17136 head
17137 );
17138 assert!(runner.state.reviews.is_empty());
17139 assert!(!runner.state.gate_ran);
17140 }
17141 let notice = crate::notices::run_ended(&runner.state).unwrap();
17142 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17143 runner.state.begin_pass();
17144 runner.state.status = RunStatus::Voting;
17145 runner.deliberate().await.unwrap();
17146 runner.vote().await.unwrap();
17147 runner.finish_after_tally().await.unwrap();
17148 assert_eq!(runner.state.status, RunStatus::Blocked);
17149 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17150 assert_eq!(
17151 crate::notices::run_ended(&runner.state).unwrap().since,
17152 notice.since
17153 );
17154 }
17155
17156 #[cfg(unix)]
17157 #[tokio::test]
17158 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17159 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17160 for detail in [
17161 "fix/retained conflicts with origin/main and did not rebase: conflict",
17162 "rebased fix/retained locally but origin refused the push",
17163 "unknown failure",
17164 ] {
17165 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17166 runner.state.record_base_sync_stop(&head);
17167 assert!(runner.state.authorize_base_sync(&task).is_err());
17168 runner.state.status = RunStatus::Voting;
17169 runner.execute_graph().await.unwrap();
17170 assert_eq!(runner.state.status, RunStatus::Blocked);
17171 assert!(!crate::handover::can_resume(&runner.state));
17172 assert_eq!(
17173 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17174 head
17175 );
17176 }
17177 }
17178
17179 #[cfg(unix)]
17180 #[tokio::test]
17181 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17182 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17183 runner.state.authorize_base_sync(&task).unwrap();
17184 assert!(runner.state.consume_base_recovery(&head));
17185 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17186 runner.state.reserve_base_operation(&head, &tip);
17187 runner.state.save().unwrap();
17188 runner.state = RunState::load(&runner.state.id).unwrap();
17189 assert!(!runner.state.consume_base_recovery(&head));
17190 runner.sync_to_base().await.unwrap();
17191 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17192 assert_eq!(runner.state.base_episode_used(), 1);
17193 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17194 assert!(
17195 runner
17196 .state
17197 .base_sync_recovery
17198 .operations
17199 .last()
17200 .unwrap()
17201 .finished
17202 );
17203 assert!(!runner.state.base_sync_blocked());
17204 runner.sync_to_base().await.unwrap();
17205 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17206 }
17207
17208 #[cfg(unix)]
17209 #[tokio::test]
17210 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17211 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17212 let original = runner
17214 .state
17215 .base_sync
17216 .as_ref()
17217 .unwrap()
17218 .conflict
17219 .clone()
17220 .unwrap();
17221 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17222 original
17223 .replace("synchronization(s)", "rebase(s)")
17224 .replace("synchronizing", "rebasing"),
17225 );
17226 runner.state.base_sync_recovery = Default::default();
17227 runner.state.record_base_sync_stop(&head);
17228 assert_eq!(
17229 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17230 crate::base_sync_recovery::StopKind::Exhausted
17231 );
17232 runner.state.authorize_base_sync(&task).unwrap();
17233 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17234 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17235 .as_object_mut()
17236 .unwrap();
17237 auth.remove("interval");
17238 auth.remove("scope");
17239 runner.state = serde_json::from_value(legacy).unwrap();
17240 assert!(runner.state.consume_base_recovery(&head));
17241 assert_eq!(runner.state.base_episode_used(), 3);
17242 runner.sync_to_base().await.unwrap();
17243 assert_eq!(runner.state.base_episode_used(), 4);
17244 runner.state.resource_block = Some(crate::run::ResourceBlock {
17245 node: "gate".into(),
17246 kind: "cache".into(),
17247 detail: "legacy wait".into(),
17248 at: Timestamp::now(),
17249 cause: crate::run::ResourceBlockCause::Unrecorded,
17250 });
17251 runner.state.save().unwrap();
17252 assert!(
17253 runner
17254 .state
17255 .base_sync_recovery
17256 .episodes
17257 .last()
17258 .unwrap()
17259 .boundary
17260 .is_none()
17261 );
17262 runner.state.resource_block = None;
17263 advance_base_fixture(&runner).await;
17264 runner.sync_to_base().await.unwrap();
17265 assert!(runner.state.base_sync_blocked());
17266 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17267 }
17268
17269 #[cfg(unix)]
17270 async fn advance_base_fixture(runner: &Runner) {
17271 let repo = &runner.state.repo;
17272 for args in [
17273 vec!["checkout", "main"],
17274 vec!["commit", "--allow-empty", "-m", "base advanced"],
17275 vec!["push", "origin", "main"],
17276 vec!["checkout", "fix/retained"],
17277 ] {
17278 git::git(repo, &args).await.unwrap();
17279 }
17280 }
17281
17282 #[cfg(unix)]
17283 fn fresh_base_interval_fixture(runner: &mut Runner) {
17284 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17285 runner.state.base_sync.as_mut().unwrap().conflict = None;
17286 runner.state.base_sync_recovery = Default::default();
17287 runner.state.status = RunStatus::Voting;
17288 }
17289
17290 #[cfg(unix)]
17291 #[tokio::test]
17292 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17293 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17294 fresh_base_interval_fixture(&mut runner);
17295 runner.state.begin_pass();
17296 runner.sync_to_base().await.unwrap();
17297 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17298 runner.state.resource_block = Some(crate::run::ResourceBlock {
17299 node: "gate".into(),
17300 kind: "build cache".into(),
17301 detail: "busy".into(),
17302 at: Timestamp::now(),
17303 cause: crate::run::ResourceBlockCause::Timeout {
17304 waited_secs: 1,
17305 limit_secs: 1,
17306 },
17307 });
17308 runner.state.save().unwrap();
17309 let queue = crate::queue::Queue::open();
17310 let mut owner = queue.get(&task).unwrap();
17311 runner.state.task_instruction = Some(owner.instruction.clone());
17312 owner.release_operator("fixture-owner", None);
17313 queue.put(&mut owner).unwrap();
17314 let approval = owner.resumption.clone().unwrap();
17315 runner.state.resource_block = None;
17316 runner.state.save().unwrap();
17317 runner.state = RunState::load(&runner.state.id).unwrap();
17318 advance_base_fixture(&runner).await;
17319 runner.sync_to_base().await.unwrap();
17320 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17321 assert_eq!(runner.state.base_episode_used(), 1);
17322 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17323 assert!(
17324 runner.state.base_sync_recovery.episodes[0]
17325 .boundary
17326 .as_ref()
17327 .unwrap()
17328 .consumed
17329 );
17330 assert!(
17331 !runner
17332 .state
17333 .renew_base_episode(&task, &approval, &head, "base")
17334 );
17335 for _ in 0..3 {
17336 owner.release_operator("fixture-owner", None);
17337 queue.put(&mut owner).unwrap();
17338 runner.state = RunState::load(&runner.state.id).unwrap();
17339 runner.state.begin_pass();
17340 runner.sync_to_base().await.unwrap();
17341 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17342 assert_eq!(runner.state.base_episode_used(), 1);
17343 }
17344 runner.state.resource_block = Some(crate::run::ResourceBlock {
17345 node: "gate".into(),
17346 kind: "disk pressure".into(),
17347 detail: "free space required".into(),
17348 at: Timestamp::now(),
17349 cause: crate::run::ResourceBlockCause::Other,
17350 });
17351 runner.state.end_pass_parked(ParkCause::Interrupt {
17352 reason: "disk pressure".into(),
17353 });
17354 runner.state.save().unwrap();
17355 owner.release_operator("fixture-owner-after-disk", None);
17356 queue.put(&mut owner).unwrap();
17357 runner.state.resource_block = None;
17358 advance_base_fixture(&runner).await;
17359 runner.sync_to_base().await.unwrap();
17360 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17361 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17362 assert_eq!(runner.state.base_episode_used(), 1);
17363 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17364 }
17365
17366 #[cfg(unix)]
17367 #[tokio::test]
17368 async fn base_recovery_continuous_advancement_stops_at_four() {
17369 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17370 fresh_base_interval_fixture(&mut runner);
17371 for spent in 1..=BASE_SYNC_ROUNDS {
17372 runner.sync_to_base().await.unwrap();
17373 assert!(!runner.state.base_sync_blocked());
17374 assert_eq!(runner.state.base_episode_used(), spent);
17375 advance_base_fixture(&runner).await;
17376 }
17377 runner.sync_to_base().await.unwrap();
17378 assert_eq!(
17379 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17380 crate::base_sync_recovery::StopKind::Exhausted
17381 );
17382 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17383 assert!(stop.unverified_sync);
17384 let action = runner.state.base_sync_next_action();
17385 assert!(action.contains("never reached its gate"));
17386 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17387 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17388 runner.state = RunState::load(&runner.state.id).unwrap();
17389 runner.sync_to_base().await.unwrap();
17390 assert_eq!(runner.state.base_episode_used(), 4);
17391 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17392 }
17393
17394 #[cfg(unix)]
17395 #[tokio::test]
17396 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17397 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17398 runner.state.ensure_base_episode();
17399 runner.state.record_base_progress(&head);
17401 runner.state.base_sync_recovery.stop = None;
17402 runner.state.record_base_sync_stop(&head);
17403 assert!(
17404 !runner
17405 .state
17406 .base_sync_recovery
17407 .stop
17408 .as_ref()
17409 .unwrap()
17410 .unverified_sync
17411 );
17412
17413 runner.state.finish_base_operation(Some(&head));
17414 runner.state.gate_ran = true;
17416 runner.state.gate_head = Some("older-head".into());
17417 runner.state.base_sync_recovery.stop = None;
17418 runner.state.record_base_sync_stop(&head);
17419 runner.state.save().unwrap();
17420 runner.state = RunState::load(&runner.state.id).unwrap();
17421 assert!(
17422 runner
17423 .state
17424 .base_sync_recovery
17425 .stop
17426 .as_ref()
17427 .unwrap()
17428 .unverified_sync
17429 );
17430
17431 runner.state.gate_head = Some(head.clone());
17432 runner.state.base_sync_recovery.stop = None;
17433 runner.state.record_base_sync_stop(&head);
17434 assert!(
17435 !runner
17436 .state
17437 .base_sync_recovery
17438 .stop
17439 .as_ref()
17440 .unwrap()
17441 .unverified_sync
17442 );
17443 assert!(
17444 !runner
17445 .state
17446 .base_sync_next_action()
17447 .contains("never reached its gate")
17448 );
17449
17450 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17452 legacy["base_sync_recovery"]["stop"]
17453 .as_object_mut()
17454 .unwrap()
17455 .remove("unverified_sync");
17456 legacy["base_sync_recovery"]["episodes"][0]
17457 .as_object_mut()
17458 .unwrap()
17459 .remove("last_sync_head");
17460 runner.state = serde_json::from_value(legacy).unwrap();
17461 assert!(
17462 !runner
17463 .state
17464 .base_sync_recovery
17465 .stop
17466 .as_ref()
17467 .unwrap()
17468 .unverified_sync
17469 );
17470 assert!(
17471 runner.state.base_sync_recovery.episodes[0]
17472 .last_sync_head
17473 .is_none()
17474 );
17475 }
17476
17477 #[cfg(unix)]
17478 #[tokio::test]
17479 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17480 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17481 runner.state.authorize_base_sync(&task).unwrap();
17482 assert!(
17483 runner
17484 .state
17485 .base_sync_next_action()
17486 .contains("Approval recorded")
17487 );
17488 assert!(
17489 runner
17490 .state
17491 .base_sync_next_action()
17492 .contains("4/4 in episode 0, 4 lifetime")
17493 );
17494 assert!(runner.state.consume_base_recovery(&head));
17495 assert!(
17496 runner
17497 .state
17498 .base_sync_next_action()
17499 .contains("0/4 in episode 1, 4 lifetime")
17500 );
17501 }
17502
17503 #[cfg(unix)]
17504 #[tokio::test]
17505 async fn base_recovery_changed_head_invalidates_without_spending() {
17506 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17507 runner.state.authorize_base_sync(&task).unwrap();
17508 git::git(
17509 &runner.state.repo,
17510 &["commit", "--allow-empty", "-m", "external head change"],
17511 )
17512 .await
17513 .unwrap();
17514 runner.sync_to_base().await.unwrap();
17515 let a = runner
17516 .state
17517 .base_sync_recovery
17518 .authorizations
17519 .last()
17520 .unwrap();
17521 assert!(a.consumed);
17522 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17523 runner.state = RunState::load(&runner.state.id).unwrap();
17524 assert!(
17525 runner
17526 .state
17527 .base_sync_next_action()
17528 .contains("Approval invalidated")
17529 );
17530 assert!(!runner.state.base_recovery_pending());
17531 assert!(runner.state.base_sync_blocked());
17532 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17533 }
17534
17535 #[cfg(unix)]
17536 #[tokio::test]
17537 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17538 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17539 runner.state.driver_exited = true;
17540 runner.state.save().unwrap();
17541 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17542 .await
17543 .unwrap();
17544 runner.state = RunState::load(&runner.state.id).unwrap();
17545 advance_base_fixture(&runner).await;
17546 runner.sync_to_base().await.unwrap();
17547 let a = runner
17548 .state
17549 .base_sync_recovery
17550 .authorizations
17551 .last()
17552 .unwrap();
17553 assert!(
17554 a.consumed
17555 && a.invalidated
17556 .as_deref()
17557 .unwrap()
17558 .contains("fetched base changed")
17559 );
17560 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17561 assert_eq!(
17562 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17563 head
17564 );
17565 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17566 .await
17567 .unwrap();
17568 runner.state = RunState::load(&runner.state.id).unwrap();
17569 runner.sync_to_base().await.unwrap();
17570 assert!(!runner.state.base_sync_blocked());
17571 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17572 }
17573
17574 #[cfg(unix)]
17575 #[tokio::test]
17576 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17577 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17578 runner.state.authorize_base_sync(&task).unwrap();
17579 assert!(runner.state.consume_base_recovery(&head));
17580 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17581 runner.state.reserve_base_operation(&head, &tip);
17582 runner.state.save().unwrap();
17583 let scratch = runner.state.dir().join("base-sync");
17584 assert_eq!(
17585 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17586 .await
17587 .unwrap(),
17588 crate::rebase::Rebased::Applied
17589 );
17590 runner.state = RunState::load(&runner.state.id).unwrap();
17591 runner.sync_to_base().await.unwrap();
17592 assert!(
17593 runner
17594 .state
17595 .base_sync_recovery
17596 .operations
17597 .last()
17598 .unwrap()
17599 .finished
17600 );
17601 assert_eq!(runner.state.base_episode_used(), 1);
17602 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17603 assert!(!runner.state.base_sync_blocked());
17604 }
17605
17606 #[cfg(unix)]
17607 #[tokio::test]
17608 async fn base_recovery_crash_preserves_untracked_files() {
17609 for ignored in [false, true] {
17610 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17611 runner.state.authorize_base_sync(&task).unwrap();
17612 assert!(runner.state.consume_base_recovery(&head));
17613 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17614 runner.state.reserve_base_operation(&head, &tip);
17615 runner.state.save().unwrap();
17616 let scratch = runner.state.dir().join("base-sync");
17617 assert_eq!(
17618 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17619 .await
17620 .unwrap(),
17621 crate::rebase::Rebased::Applied
17622 );
17623 let repo = &runner.state.repo;
17624 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17625 assert_ne!(result, head);
17626 if ignored {
17627 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17628 .await
17629 .unwrap();
17630 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17631 }
17632 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17633 assert!(
17634 git::git_raw(repo, &["diff", "--quiet", &head])
17635 .await
17636 .unwrap()
17637 .ok()
17638 );
17639 runner.state = RunState::load(&runner.state.id).unwrap();
17641 runner.sync_to_base().await.unwrap();
17642 assert!(runner.state.base_sync_blocked());
17643 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17644 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17645 assert_eq!(
17646 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17647 result
17648 );
17649 assert_eq!(
17650 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17651 "retained work\n"
17652 );
17653 assert!(
17654 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17655 .await
17656 .unwrap()
17657 .ok()
17658 );
17659 }
17660 }
17661
17662 #[cfg(unix)]
17663 #[tokio::test]
17664 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17665 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17666 runner.state.authorize_base_sync(&task).unwrap();
17667 assert!(runner.state.consume_base_recovery(&head));
17668 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17669 runner.state.reserve_base_operation(&head, &tip);
17670 runner.state.save().unwrap();
17671 let scratch = runner.state.dir().join("base-sync");
17672 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17673 .await
17674 .unwrap();
17675 git::sync_to_head(&runner.state.repo).await.unwrap();
17676 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17677 git::git(&runner.state.repo, &["add", "foreign"])
17678 .await
17679 .unwrap();
17680 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17681 .await
17682 .unwrap();
17683 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17684 runner.state = RunState::load(&runner.state.id).unwrap();
17685 runner.sync_to_base().await.unwrap();
17686 assert!(runner.state.base_sync_blocked());
17687 assert!(
17688 !runner
17689 .state
17690 .base_sync_recovery
17691 .operations
17692 .last()
17693 .unwrap()
17694 .finished
17695 );
17696 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17697 assert_eq!(
17698 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17699 foreign_head
17700 );
17701 }
17702
17703 #[cfg(unix)]
17704 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17705 {
17706 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17707 fresh_base_interval_fixture(&mut runner);
17708 let repo = &runner.state.repo;
17709 git::git(repo, &["checkout", "main"]).await.unwrap();
17710 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17711 git::git(repo, &["add", "retained"]).await.unwrap();
17712 git::git(repo, &["commit", "-m", "upstream retained change"])
17713 .await
17714 .unwrap();
17715 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17716 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17717 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17718 let scratch = runner.state.dir().join("base-sync");
17719 assert!(matches!(
17720 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17721 .await
17722 .unwrap(),
17723 git::RebaseStart::Conflicted(_)
17724 ));
17725 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17726 agent: "fixture".into(),
17727 paths: vec!["retained".into()],
17728 from: Some(head.clone()),
17729 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17730 index_outside: None,
17731 finished: false,
17732 error: None,
17733 launcher: None,
17734 infra: None,
17735 outage: None,
17736 });
17737 (tmp, runner, head, tip, scratch)
17738 }
17739
17740 #[cfg(unix)]
17741 #[tokio::test]
17742 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17743 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17744 std::fs::write(
17747 scratch.join("retained"),
17748 "upstream and retained resolution\n",
17749 )
17750 .unwrap();
17751 runner.state.resource_block = Some(crate::run::ResourceBlock {
17752 node: "rebase".into(),
17753 kind: "repository coordination".into(),
17754 detail: "busy".into(),
17755 at: Timestamp::now(),
17756 cause: crate::run::ResourceBlockCause::Other,
17757 });
17758 runner.state.save().unwrap();
17759 let newer = git::git(
17762 &runner.state.repo,
17763 &[
17764 "commit-tree",
17765 "main^{tree}",
17766 "-p",
17767 "main",
17768 "-m",
17769 "base advanced during wait",
17770 ],
17771 )
17772 .await
17773 .unwrap();
17774 git::git(
17775 &runner.state.repo,
17776 &[
17777 "push",
17778 "origin",
17779 &format!("{}:refs/heads/main", newer.trim()),
17780 ],
17781 )
17782 .await
17783 .unwrap();
17784 runner.state = RunState::load(&runner.state.id).unwrap();
17785 runner.state.resource_block = None;
17786 runner.sync_to_base().await.unwrap();
17787 assert!(!runner.state.base_sync_blocked());
17788 let op = &runner.state.base_sync_recovery.operations[0];
17789 assert_eq!(op.head, head);
17790 assert_eq!(op.tip, tip);
17791 assert!(op.finished);
17792 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17793 assert_eq!(runner.state.rebase_fixes.len(), 1);
17794 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17795 .await
17796 .unwrap();
17797 assert_ne!(latest, tip);
17798 runner.state = RunState::load(&runner.state.id).unwrap();
17799 runner.sync_to_base().await.unwrap();
17800 assert!(!runner.state.base_sync_blocked());
17801 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17802 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17803 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17804 }
17805
17806 #[cfg(unix)]
17807 #[tokio::test]
17808 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17809 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17810 runner.state.reserve_base_operation(&head, &tip);
17811 runner.state.save().unwrap();
17812 std::fs::write(
17813 scratch.join("retained"),
17814 "upstream and retained resolution\n",
17815 )
17816 .unwrap();
17817 git::git(&scratch, &["add", "retained"]).await.unwrap();
17818 git::git(
17819 &scratch,
17820 &["-c", "core.editor=true", "rebase", "--continue"],
17821 )
17822 .await
17823 .unwrap();
17824 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17825 assert!(
17826 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17827 .await
17828 .unwrap()
17829 .is_empty()
17830 );
17831 assert_eq!(
17832 crate::rebase::finish(
17833 &mut runner.state,
17834 &scratch,
17835 "fix/retained",
17836 &head,
17837 &tip,
17838 &["retained".into()],
17839 ""
17840 )
17841 .await
17842 .unwrap(),
17843 crate::rebase::Rebased::Applied
17844 );
17845 assert!(!scratch.exists());
17846 runner.state = RunState::load(&runner.state.id).unwrap();
17848 let op = &runner.state.base_sync_recovery.operations[0];
17849 assert!(!op.finished);
17850 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17851 runner.sync_to_base().await.unwrap();
17852 assert!(!runner.state.base_sync_blocked());
17853 assert!(runner.state.base_sync_recovery.operations[0].finished);
17854 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17855 assert_eq!(runner.state.rebase_fixes.len(), 1);
17856 let winner = runner.state.winner().unwrap().clone();
17857 let wait = runner.repository_wait();
17858 assert!(
17859 verify_current_head(&mut runner.state, &winner, &wait)
17860 .await
17861 .unwrap()
17862 );
17863 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17864 }
17865
17866 #[cfg(unix)]
17867 #[tokio::test]
17868 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17869 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17870 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17871 runner.state.reserve_base_operation(&head, &tip);
17872 runner.state.save().unwrap();
17873 std::fs::write(
17874 scratch.join("retained"),
17875 "upstream and retained resolution\n",
17876 )
17877 .unwrap();
17878 git::git(&scratch, &["add", "retained"]).await.unwrap();
17879 git::git(
17880 &scratch,
17881 &["-c", "core.editor=true", "rebase", "--continue"],
17882 )
17883 .await
17884 .unwrap();
17885 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17886 assert!(!git::rebase_in_progress(&scratch).await);
17887 assert!(
17888 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17889 .await
17890 .unwrap()
17891 .is_empty()
17892 );
17893 match damage {
17894 Some("record") => {
17895 let path =
17896 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17897 .await
17898 .unwrap();
17899 std::fs::write(scratch.join(path), "").unwrap();
17900 }
17901 Some("foreign") => {
17902 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17903 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17904 git::git(&scratch, &["commit", "-m", "foreign work"])
17905 .await
17906 .unwrap();
17907 }
17908 Some("dirty") => {
17909 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17910 }
17911 _ => {}
17912 }
17913 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17914 runner.state = RunState::load(&runner.state.id).unwrap();
17916 assert!(
17917 runner.state.base_sync_recovery.operations[0]
17918 .validated_head
17919 .is_none()
17920 );
17921 runner.sync_to_base().await.unwrap();
17922 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17923 assert_eq!(runner.state.base_episode_used(), 1);
17924 assert_eq!(runner.state.rebase_fixes.len(), 1);
17925 assert_eq!(
17926 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17927 retained_head
17928 );
17929 if damage.is_some() {
17930 assert!(runner.state.base_sync_blocked());
17931 assert!(scratch.exists());
17932 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17933 assert!(
17934 runner.state.base_sync_recovery.operations[0]
17935 .validated_head
17936 .is_none()
17937 );
17938 } else {
17939 assert!(!runner.state.base_sync_blocked());
17940 assert!(!scratch.exists());
17941 assert!(runner.state.base_sync_recovery.operations[0].finished);
17942 assert_eq!(
17943 runner.state.base_sync_recovery.operations[0]
17944 .validated_head
17945 .as_deref(),
17946 Some(result.as_str())
17947 );
17948 let winner = runner.state.winner().unwrap().clone();
17949 let wait = runner.repository_wait();
17950 assert!(
17951 verify_current_head(&mut runner.state, &winner, &wait)
17952 .await
17953 .unwrap()
17954 );
17955 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17956 runner.state = RunState::load(&runner.state.id).unwrap();
17957 runner.sync_to_base().await.unwrap();
17958 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17959 }
17960 }
17961 }
17962
17963 #[cfg(unix)]
17964 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17965 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17966 runner.state.prep_complete = Some(true);
17968 runner.state.base_sync = None;
17969 runner.state.base_sync_recovery = Default::default();
17970 runner.state.config.graph.gate_fix_rounds = 1;
17971 let mut review = review_round(true, 0, 1, 1, false, true);
17972 review.head = head.clone();
17973 review.verified_head = Some(head.clone());
17974 runner.state.reviews = vec![review];
17975 runner.state.gate = vec![CommandOutcome {
17976 command: "old gate".into(),
17977 code: Some(1),
17978 output_tail: "old failure".into(),
17979 duration_ms: 0,
17980 resource_blocked: false,
17981 }];
17982 runner.state.gate_head = Some(head.clone());
17983 runner.state.gate_ran = true;
17984 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17985 agent: "fixture".into(),
17986 failed: runner.state.gate.clone(),
17987 notes: "spent".into(),
17988 committed: false,
17989 error: None,
17990 }];
17991 runner.state.land_approval = Some(crate::run::LandApproval {
17992 question: "old approval".into(),
17993 head: head.clone(),
17994 });
17995 runner.state.status = RunStatus::Blocked;
17996 (tmp, runner, head)
17997 }
17998
17999 #[cfg(unix)]
18000 #[tokio::test]
18001 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18002 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18003 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18004 let phases = tmp.path().join("phases");
18005 runner.state.config.verify.e2e = vec![format!(
18006 "echo e2e >> '{}'; {}",
18007 phases.display(),
18008 if e2e_ok {
18009 "true"
18010 } else {
18011 "echo new-e2e-failure; false"
18012 }
18013 )];
18014 runner.state.config.verify.gate = vec![format!(
18015 "echo gate >> '{}'; {}",
18016 phases.display(),
18017 if gate_ok {
18018 "true"
18019 } else {
18020 "echo new-gate-failure; false"
18021 }
18022 )];
18023 runner.state.save().unwrap();
18024 runner.state = RunState::load(&runner.state.id).unwrap();
18025 runner.sync_to_base().await.unwrap();
18026 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18027 assert_ne!(head, old);
18028 runner.state = RunState::load(&runner.state.id).unwrap();
18030 runner.execute_graph().await.unwrap();
18031 assert_eq!(runner.state.gate_fixes.len(), 1);
18032 assert_eq!(runner.state.reviews.len(), 1);
18033 assert_eq!(runner.state.reviews[0].head, old);
18034 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18035 assert_eq!(runner.state.verification_history.len(), 1);
18036 let history = &runner.state.verification_history[0];
18037 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18038 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18039 assert!(history.archived_at.is_some());
18040 assert_eq!(history.fixes_spent, Some(1));
18041 assert_eq!(history.gate[0].output_tail, "old failure");
18042 let archive_events: Vec<_> = runner
18043 .state
18044 .events
18045 .iter()
18046 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18047 .collect();
18048 assert_eq!(archive_events.len(), 1);
18049 assert!(archive_events[0].message.contains(&short(&old)));
18050 assert!(archive_events[0].message.contains(&short(&head)));
18051 assert!(
18052 archive_events[0]
18053 .message
18054 .contains("1 gate-fix round(s) already spent stay spent")
18055 );
18056 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18057 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18058 if e2e_ok {
18059 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18060 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18061 } else {
18062 assert!(!runner.state.gate_ran);
18063 }
18064 assert_eq!(
18065 runner.state.status,
18066 if e2e_ok && gate_ok {
18067 RunStatus::Ready
18068 } else {
18069 RunStatus::Blocked
18070 }
18071 );
18072 if !e2e_ok || !gate_ok {
18073 let cause = runner.state.verification_blocking_cause().unwrap();
18074 assert!(cause.contains(&head));
18075 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18076 runner.state.event("land", "rebased successfully");
18078 let notice = crate::notices::run_ended(&runner.state).unwrap();
18079 assert!(!format!("{notice:?}").contains("rebased successfully"));
18080 assert!(runner.state.merge.is_none());
18081 }
18082 let before = std::fs::read_to_string(&phases).unwrap();
18083 runner.state.save().unwrap();
18084 runner.state = RunState::load(&runner.state.id).unwrap();
18085 runner.review_loop().await.unwrap();
18086 runner.gate().await.unwrap();
18087 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18088 assert_eq!(runner.state.verification_history.len(), 1);
18089 let history = &runner.state.verification_history[0];
18090 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18091 assert!(history.archived_at.is_some());
18092 assert_eq!(history.fixes_spent, Some(1));
18093 }
18094 }
18095
18096 #[cfg(unix)]
18097 #[tokio::test]
18098 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18099 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18100 let marker = tmp.path().join("gate-ran");
18101 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18102 runner.review_loop().await.unwrap();
18103 runner.gate().await.unwrap();
18104 assert_eq!(runner.state.status, RunStatus::Blocked);
18105 assert!(!marker.exists());
18106 assert!(runner.state.verification_history.is_empty());
18107 runner.state.gate_head = None;
18108 runner.gate().await.unwrap();
18109 assert!(marker.exists());
18110 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18111 assert_eq!(runner.state.verification_history.len(), 1);
18112 assert_eq!(runner.state.gate_fixes.len(), 1);
18113 }
18114
18115 #[cfg(unix)]
18116 #[tokio::test]
18117 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18118 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18119 runner.sync_to_base().await.unwrap();
18120 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18121 runner.state.reconcile_verification_head(&head);
18122 for outcomes in [
18124 Vec::new(),
18125 vec![CommandOutcome {
18126 command: "true".into(),
18127 code: None,
18128 output_tail: "cache unavailable".into(),
18129 duration_ms: 0,
18130 resource_blocked: true,
18131 }],
18132 ] {
18133 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18134 head: head.clone(),
18135 passed: false,
18136 outcomes,
18137 });
18138 runner.state.gate_ran = false;
18139 runner.state.gate.clear();
18140 runner.state.save().unwrap();
18141 runner.state = RunState::load(&runner.state.id).unwrap();
18142 runner.merge().await.unwrap();
18143 assert!(runner.state.merge.is_none());
18144 runner.review_loop().await.unwrap();
18145 runner.gate().await.unwrap();
18146 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18147 assert!(runner.state.gate_allows_progress());
18148 assert_eq!(runner.state.gate_fixes.len(), 1);
18149 assert!(runner.state.merge.is_none());
18150 }
18151 git::git(
18153 &runner.state.repo,
18154 &["commit", "--allow-empty", "-m", "external update"],
18155 )
18156 .await
18157 .unwrap();
18158 runner.merge().await.unwrap();
18159 assert!(runner.state.merge.is_none());
18160 assert_eq!(runner.state.status, RunStatus::Blocked);
18161 }
18162
18163 #[cfg(unix)]
18164 #[tokio::test]
18165 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18166 for waiting_on_gate in [false, true] {
18167 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18168 runner.sync_to_base().await.unwrap();
18169 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18170 runner.state.reconcile_verification_head(&head);
18171 if waiting_on_gate {
18172 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18173 head: head.clone(),
18174 passed: true,
18175 outcomes: Vec::new(),
18176 });
18177 }
18178 let cache = tmp.path().join("cache");
18179 runner.state.config.verify.e2e =
18180 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18181 runner.state.config.verify.gate = vec!["true".into()];
18182 runner.state.config.graph.timeout_resource_wait = 1;
18183 let owner =
18184 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18185 let guard =
18186 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18187 crate::cache::AcquireOutcome::Acquired(g) => g,
18188 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18189 };
18190 runner.review_loop().await.unwrap();
18191 runner.gate().await.unwrap();
18192 assert!(!runner.state.gate_ran);
18193 assert_eq!(runner.state.gate_fixes.len(), 1);
18194 assert!(runner.state.resource_block.is_some());
18195 runner.merge().await.unwrap();
18196 assert!(runner.state.merge.is_none());
18197 drop(guard);
18198 runner.state = RunState::load(&runner.state.id).unwrap();
18199 runner.state.resource_block = None;
18201 runner.review_loop().await.unwrap();
18202 runner.gate().await.unwrap();
18203 assert!(runner.state.gate_allows_progress());
18204 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18205 assert_eq!(runner.state.gate_fixes.len(), 1);
18206 }
18207 }
18208
18209 #[cfg(unix)]
18210 #[tokio::test]
18211 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18212 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18213 runner.sync_to_base().await.unwrap();
18214 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18215 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18216 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18218 runner.review_loop().await.unwrap();
18219 runner.gate().await.unwrap();
18220 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18221 assert_ne!(before, after);
18222 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18223 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18224 assert!(!runner.state.gate_allows_progress());
18225 runner.merge().await.unwrap();
18226 assert!(runner.state.merge.is_none());
18227 assert!(
18228 runner
18229 .state
18230 .verification_history
18231 .iter()
18232 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18233 );
18234
18235 runner.state.reviews[0].clean = false;
18237 runner.state.reviews[0].answered = 0;
18238 runner.state.config.verify.e2e = vec!["true".into()];
18239 runner.review_loop().await.unwrap();
18240 runner.gate().await.unwrap();
18241 runner.merge().await.unwrap();
18242 assert_eq!(runner.state.status, RunStatus::Blocked);
18243 assert!(runner.state.merge.is_none());
18244 assert!(
18245 runner
18246 .state
18247 .verification_blocking_cause()
18248 .unwrap()
18249 .contains("review blocked")
18250 );
18251 }
18252
18253 #[cfg(unix)]
18254 #[tokio::test]
18255 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18256 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18257 runner.state.gate_head = None;
18258 runner.state.config.verify.gate =
18259 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18260 runner.review_loop().await.unwrap();
18261 runner.gate().await.unwrap();
18262 assert_ne!(
18263 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18264 old
18265 );
18266 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18267 assert!(!runner.state.gate_allows_progress());
18268 assert_eq!(runner.state.status, RunStatus::Blocked);
18269 runner.merge().await.unwrap();
18270 assert!(runner.state.merge.is_none());
18271 }
18272
18273 #[cfg(unix)]
18274 #[tokio::test]
18275 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18276 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18277 let mut old_review = review_round(true, 0, 1, 1, false, true);
18278 old_review.head = head.clone();
18279 old_review.verified_head = Some(head.clone());
18280 runner.state.reviews = vec![old_review];
18281 runner.state.gate = vec![CommandOutcome {
18282 command: "old gate".into(),
18283 code: None,
18284 output_tail: "timed out after 2400s".into(),
18285 duration_ms: 2_400_000,
18286 resource_blocked: false,
18287 }];
18288 runner.state.gate_ran = true;
18289 runner.state.gate_head = Some(head.clone());
18290 runner.state.land_approval = Some(crate::run::LandApproval {
18291 question: "old-head-approval".into(),
18292 head: head.clone(),
18293 });
18294 runner.state.authorize_base_sync(&task).unwrap();
18295 runner.sync_to_base().await.unwrap();
18296 assert!(!runner.state.base_sync_blocked());
18297 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18298 assert_ne!(head, fresh);
18299 assert_eq!(runner.state.reviews.len(), 1);
18300 assert!(!runner.state.gate_ran);
18301 let saved = &runner.state.base_sync_recovery.authorizations[0];
18302 assert_eq!(saved.reviews[0].head, head);
18303 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18304 assert!(saved.gate_ran);
18305 assert!(!saved.gate[0].ok());
18306 runner.review_loop().await.unwrap();
18307 runner.gate().await.unwrap();
18308 assert_eq!(runner.state.reviews[0].head, head);
18309 assert_eq!(
18310 runner.state.reviews[0].verified_head.as_deref(),
18311 Some(head.as_str())
18312 );
18313 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18314 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18315 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18316 assert!(runner.state.gate_ran);
18317 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18318 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18319 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18320 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18321 assert_eq!(
18322 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18323 "retained change"
18324 );
18325 }
18326
18327 #[cfg(unix)]
18328 #[tokio::test]
18329 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18330 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18331 let queue = crate::queue::Queue::open();
18332 let mut owner = queue.get(&task).unwrap();
18333 owner.release_operator("test-resume", None);
18334 queue.put(&mut owner).unwrap();
18335 let generation = owner.resumption.as_ref().unwrap().generation;
18336 runner.state.task_instruction = Some(owner.instruction.clone());
18337 runner.state.driver_exited = true;
18338 runner.state.save().unwrap();
18339 let home = crate::run::home();
18340 let _claim = queue.claim(&task).unwrap();
18341 owner.resumption.as_mut().unwrap().at =
18342 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18343 queue.put(&mut owner).unwrap();
18344 assert!(
18345 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18346 .await
18347 .is_err(),
18348 "an approval from before the stop grants nothing"
18349 );
18350 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18351 queue.put(&mut owner).unwrap();
18352 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18353 .await
18354 .unwrap();
18355 assert!(
18356 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18357 .await
18358 .is_err()
18359 );
18360 runner.state = RunState::load(&runner.state.id).unwrap();
18361 assert_eq!(
18362 runner.state.base_sync_recovery.authorizations[0].generation,
18363 Some(generation)
18364 );
18365 runner.sync_to_base().await.unwrap();
18366 let repo = runner.state.repo.clone();
18367 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18368 git::git(&repo, &["checkout", "main"]).await.unwrap();
18369 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18370 git::git(&repo, &["add", "."]).await.unwrap();
18371 git::git(&repo, &["commit", "-m", "move base again"])
18372 .await
18373 .unwrap();
18374 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18375 git::git(&repo, &["checkout", "fix/retained"])
18376 .await
18377 .unwrap();
18378 runner.sync_to_base().await.unwrap();
18379 assert!(!runner.state.base_sync_blocked());
18380 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18381 assert_eq!(runner.state.base_episode_used(), 2);
18382 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18383 assert_ne!(fresh, head);
18384 assert!(!runner.state.base_recovery_pending());
18385 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18387 queue.put(&mut owner).unwrap();
18388 assert!(
18389 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18390 .await
18391 .is_err()
18392 );
18393 runner.state = RunState::load(&runner.state.id).unwrap();
18394 runner.sync_to_base().await.unwrap();
18395 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18396 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18397 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18398 }
18399
18400 #[cfg(unix)]
18401 #[tokio::test]
18402 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18403 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18404 runner.state.base_sync = None;
18405 runner.state.status = RunStatus::Blocked;
18406 let mut second = runner.state.candidates[0].clone();
18407 second.label = 'B';
18408 runner.state.candidates.push(second);
18409 assert!(runner.state.votes.is_empty());
18410 assert!(runner.state.deliberation.is_empty());
18411 runner.deliberate().await.unwrap();
18412 runner.vote().await.unwrap();
18413 assert_eq!(runner.state.status, RunStatus::Blocked);
18414 assert!(runner.state.votes.is_empty());
18415 assert!(runner.state.deliberation.is_empty());
18416 }
18417
18418 fn runner_at(status: RunStatus) -> Runner {
18421 let mut state = RunState::new(
18422 PathBuf::from("/nonexistent/repo"),
18423 "main".to_owned(),
18424 "deadbeef".to_owned(),
18425 "task".to_owned(),
18426 Config::default(),
18427 );
18428 state.config.disk.min_free_bytes = 0;
18430 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18433 state.id = format!(
18434 "20000101-000000-{:04x}",
18435 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18436 );
18437 state.status = status;
18438 Runner {
18439 state,
18440 roles: ResolvedRoles {
18441 implementers: Vec::new(),
18442 judges: Vec::new(),
18443 reviewers: Vec::new(),
18444 fixer: None,
18445 conductor: conductor(),
18446 implementer_roster: Vec::new(),
18447 judge_roster: Vec::new(),
18448 reviewer_roster: Vec::new(),
18449 },
18450 sem: Arc::new(Semaphore::new(1)),
18451 pause: Pause::new(),
18452 interrupt: Pause::new(),
18453 }
18454 }
18455
18456 #[tokio::test]
18457 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18458 crate::run::pin_test_home();
18459 let temp = tempfile::tempdir().unwrap();
18460 let root = temp.path().canonicalize().unwrap();
18461 let repo = root.join("repo");
18462 std::fs::create_dir(&repo).unwrap();
18463 init_repo(&repo);
18464 git::git(&repo, &["config", "core.autocrlf", "false"])
18465 .await
18466 .unwrap();
18467 let mut runner = runner_at(RunStatus::Blocked);
18468 runner.state.repo = repo.clone();
18469 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18470 runner.state.config.agents = vec![conductor()];
18471 runner.state.config.graph.selected_implementer = Some("conductor".into());
18472 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18473 let (shell, command, setup_output) = if cfg!(windows) {
18474 (
18475 vec!["cmd".into(), "/C".into()],
18476 "echo ready>seat-ready",
18477 "ready\r\n",
18478 )
18479 } else {
18480 (
18481 vec!["sh".into(), "-c".into()],
18482 "printf ready > seat-ready",
18483 "ready",
18484 )
18485 };
18486 runner.state.config.verify.shell = Some(shell);
18487 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18488 run: Some(command.into()),
18489 ..Default::default()
18490 }];
18491 runner.state.save().unwrap();
18492 let path = runner.state.worktree_root().join("judge-1");
18493 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18494 .await
18495 .unwrap();
18496 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18497 git::git(&path, &["add", "."]).await.unwrap();
18498 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18499 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18500 .await
18501 .unwrap();
18502 git::git(&path, &["checkout", "--detach", &head])
18503 .await
18504 .unwrap();
18505 let evidence = crate::fold_evidence::snapshot(
18506 &runner.state,
18507 &crate::run::home().canonicalize().unwrap(),
18508 &head,
18509 64 * 1024 * 1024,
18510 )
18511 .await
18512 .unwrap();
18513 let mut contents = BTreeMap::new();
18514 contents.insert(
18515 repo.join("logs/HEAD"),
18516 format!("{head} {head} fixture\n").into_bytes(),
18517 );
18518 let record = serde_json::json!({
18519 "version":1, "run":runner.state.id, "source":path, "head":head,
18520 "evidence":evidence, "admin":{}, "admin_contents":contents,
18521 "history":{head.clone():evidence}, "phase":"evicted"
18522 });
18523 std::fs::write(
18524 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18525 serde_json::to_vec(&record).unwrap(),
18526 )
18527 .unwrap();
18528 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18529 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18530 .await
18531 .unwrap();
18532 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18533 assert!(
18534 !git::git_raw(&repo, &["cat-file", "-e", &head])
18535 .await
18536 .unwrap()
18537 .ok()
18538 );
18539 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18540 let error = resumed.execute().await.unwrap_err();
18541 assert!(
18542 error
18543 .to_string()
18544 .contains("selected implementation is held"),
18545 "{error:#}"
18546 );
18547 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18548 assert_eq!(
18549 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18550 setup_output
18551 );
18552 assert_eq!(
18553 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18554 "unrelated seat"
18555 );
18556 let error = resumed.execute().await.unwrap_err();
18557 assert!(
18558 error
18559 .to_string()
18560 .contains("selected implementation is held"),
18561 "{error:#}"
18562 );
18563 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18564 git::git(&path, &["add", "."]).await.unwrap();
18565 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18566 let next = git::git(
18567 &path,
18568 &[
18569 "commit-tree",
18570 &next_tree,
18571 "-p",
18572 &head,
18573 "-m",
18574 "next review checkpoint",
18575 ],
18576 )
18577 .await
18578 .unwrap();
18579 git::git(&path, &["checkout", "--detach", &next])
18580 .await
18581 .unwrap();
18582 resumed.state.reviews.push(
18583 serde_json::from_value(serde_json::json!({
18584 "round":1, "head":next, "reviews":[]
18585 }))
18586 .unwrap(),
18587 );
18588 resumed.state.save().unwrap();
18589 let error = resumed.execute().await.unwrap_err();
18590 assert!(
18591 error
18592 .to_string()
18593 .contains("selected implementation is held"),
18594 "{error:#}"
18595 );
18596 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18597 }
18598
18599 #[tokio::test]
18600 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18601 crate::run::pin_test_home();
18602 let mut runner = runner_at(RunStatus::Gating);
18603 runner.state.save().unwrap();
18604 let id = runner.state.id.clone();
18605 let passes = runner.state.passes.len();
18606 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18607 let mut claiming = Box::pin(runner.claim_driver());
18608 std::future::poll_fn(|cx| {
18609 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18610 std::task::Poll::Ready(())
18611 })
18612 .await;
18613 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18614 drop(owner);
18615 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18616 .await
18617 .unwrap()
18618 .unwrap();
18619 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18620 assert!(execution.runner.state.resource_block.is_none());
18621 assert_eq!(
18622 execution.runner.state.liveness(false),
18623 crate::run::Liveness::Live
18624 );
18625 drop(execution);
18626 assert!(RunState::load(&id).unwrap().driver_exited);
18627 }
18628
18629 #[tokio::test]
18630 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18631 crate::run::pin_test_home();
18632 let mut runner = runner_at(RunStatus::Gating);
18633 runner.state.save().unwrap();
18634 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18637 assert!(runner.claim_driver().await.is_err());
18638 assert!(runner.state.driver_exited);
18639 assert_eq!(runner.state.driver_pid, None);
18640 let disk = RunState::load(&runner.state.id).unwrap();
18641 assert!(disk.driver_exited);
18642 assert_eq!(disk.driver_pid, None);
18643 assert_eq!(disk.status, RunStatus::Gating);
18644 }
18645
18646 #[tokio::test]
18647 async fn an_early_graph_error_releases_its_driver() {
18648 crate::run::pin_test_home();
18649 let mut runner = runner_at(RunStatus::Blocked);
18650 runner.state.config.graph.selected_implementer = Some("selected".into());
18651 runner.state.save().unwrap();
18652 assert!(runner.execute().await.is_err());
18653 let disk = RunState::load(&runner.state.id).unwrap();
18654 assert!(disk.driver_exited);
18655 assert_eq!(disk.driver_pid, None);
18656 assert_eq!(disk.status, RunStatus::Blocked);
18657 }
18658
18659 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18660 crate::run::MergeRepair {
18661 question: "repair-question".into(),
18662 head: "old-head".into(),
18663 pr: "https://github.com/o/r/pull/42".into(),
18664 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18665 gate: vec![outcome(Some(0), false)],
18666 gate_head: Some("old-head".into()),
18667 gate_fixes: Vec::new(),
18668 pre_gate: Vec::new(),
18669 phase,
18670 failure: None,
18671 }
18672 }
18673
18674 #[tokio::test]
18675 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18676 crate::run::pin_test_home();
18677 let mut runner = runner_at(RunStatus::Landing);
18678 runner.state.merge_repairs.push(merge_repair_request(
18679 crate::run::MergeRepairPhase::Requested,
18680 ));
18681 runner.state.save().unwrap();
18682 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18683 assert!(
18684 runner
18685 .execute()
18686 .await
18687 .unwrap_err()
18688 .to_string()
18689 .contains("already running")
18690 );
18691 assert_eq!(
18692 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18693 crate::run::MergeRepairPhase::Requested
18694 );
18695 drop(claim);
18696 let mut disk = runner.state.clone();
18697 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18698 disk.save().unwrap();
18699 assert!(
18700 runner
18701 .execute()
18702 .await
18703 .unwrap_err()
18704 .to_string()
18705 .contains("decision changed")
18706 );
18707 assert_eq!(
18708 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18709 crate::run::MergeRepairPhase::Complete
18710 );
18711 }
18712
18713 #[test]
18714 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18715 crate::run::pin_test_home();
18716 let mut runner = runner_at(RunStatus::Landing);
18717 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18718 runner.state.reviews = request.reviews.clone();
18719 runner.state.gate = request.gate.clone();
18720 runner.state.gate_ran = true;
18721 runner.state.land_approval = Some(crate::run::LandApproval {
18722 question: "old".into(),
18723 head: "old-head".into(),
18724 });
18725 runner.state.merge_repairs.push(request);
18726 runner.begin_merge_repair_review(0).unwrap();
18727 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18728 assert!(!runner.state.gate_status().ok());
18729 assert!(runner.state.land_approval.is_none());
18730 let disk = RunState::load(&runner.state.id).unwrap();
18731 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18732 assert!(disk.merge_repairs[0].gate[0].ok());
18733 assert_eq!(
18734 disk.merge_repairs[0].phase,
18735 crate::run::MergeRepairPhase::Reviewing
18736 );
18737 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18738 assert!(disk.reviews.is_empty());
18739 }
18740
18741 #[tokio::test]
18742 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18743 crate::run::pin_test_home();
18744 for phase in [
18745 crate::run::MergeRepairPhase::Fixing,
18746 crate::run::MergeRepairPhase::Requested,
18747 ] {
18748 let mut runner = runner_at(RunStatus::Landing);
18749 runner.state.config.graph.land_rounds = 0;
18750 runner.state.merge_repairs.push(merge_repair_request(phase));
18751 runner.state.save().unwrap();
18752 assert!(!runner.repair_merge_review().await.unwrap());
18753 assert_eq!(runner.state.status, RunStatus::Blocked);
18754 assert_eq!(
18755 runner.state.merge_repairs[0].phase,
18756 crate::run::MergeRepairPhase::Failed
18757 );
18758 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18759 assert!(
18760 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18761 "interrupted"
18762 } else {
18763 "budget exhausted"
18764 })
18765 );
18766 assert!(runner.state.merge.is_none());
18767 }
18768 }
18769
18770 #[test]
18774 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18775 crate::run::pin_test_home();
18776 let mut runner = runner_at(RunStatus::Implementing);
18777 let interrupt = Pause::new();
18778 runner.watch_interrupt(interrupt.clone());
18779
18780 interrupt.park_because("task a1b2 asked to run first");
18781
18782 assert!(runner.park_here().expect("park_here"));
18783 assert!(runner.state.parked);
18784 let last = runner.state.events.last().expect("a park event");
18785 assert_eq!(last.node, "park");
18786 assert!(
18787 last.message.contains("task a1b2 asked to run first"),
18788 "expected the interrupt reason in {:?}",
18789 last.message
18790 );
18791 }
18792
18793 #[test]
18796 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18797 crate::run::pin_test_home();
18798 let mut runner = runner_at(RunStatus::Implementing);
18799 runner.state.begin_pass();
18800 let shutdown = Pause::new();
18801 runner.on_pause(shutdown.clone());
18802 let upgrade = ParkCause::Upgrade {
18803 from: "0.121.0".to_owned(),
18804 to: Some("0.121.1".to_owned()),
18805 };
18806 shutdown.park_for(upgrade.clone());
18807 shutdown.park_for(ParkCause::Operator);
18808
18809 assert!(runner.park_here().expect("park_here"));
18810 assert_eq!(
18811 runner.state.passes[0].end,
18812 Some(crate::run::PassEnd::Parked { cause: upgrade })
18813 );
18814 let last = runner.state.events.last().expect("a park event");
18815 assert!(last.message.contains("0.121.0"), "{}", last.message);
18816 }
18817
18818 #[tokio::test]
18819 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18820 crate::run::pin_test_home();
18821 let mut runner = runner_at(RunStatus::Reviewing);
18822 runner.state.save().unwrap();
18823 let mut execution = runner.claim_driver().await.expect("first execution");
18824 assert_eq!(execution.runner.state.passes.len(), 1);
18825 execution
18826 .driver
18827 .finish(&mut execution.runner.state)
18828 .await
18829 .unwrap();
18830 }
18831
18832 #[tokio::test]
18833 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18834 crate::run::pin_test_home();
18835 let mut runner = runner_at(RunStatus::Reviewing);
18836 runner.state.config.agents = vec![conductor()];
18837 runner.state.config.graph.timeout_resource_wait = 0;
18838 runner.state.save().unwrap();
18839 let id = runner.state.id.clone();
18840 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18841 assert!(runner.claim_driver().await.is_err());
18842 drop(owner);
18843 drop(runner);
18844 let mut resumed = Runner::resume(&id).unwrap();
18845 assert_eq!(resumed.state.driver_claimed, Some(false));
18846 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18847 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18848 let mut execution = resumed
18849 .claim_driver()
18850 .await
18851 .expect("retry initial claim after reload");
18852 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18853 assert_eq!(execution.runner.state.passes.len(), 1);
18854 execution
18855 .driver
18856 .finish(&mut execution.runner.state)
18857 .await
18858 .unwrap();
18859 }
18860
18861 #[tokio::test]
18862 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18863 crate::run::pin_test_home();
18864 let mut runner = runner_at(RunStatus::Reviewing);
18865 runner.state.config.agents = vec![conductor()];
18866 runner.state.save().unwrap();
18867 let id = runner.state.id.clone();
18868 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18869 let mut resumed = Runner::resume(&id).unwrap();
18870 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18871 assert!(resumed.claim_driver().await.is_err());
18872 assert!(RunState::load(&id).unwrap().passes.is_empty());
18873 drop(lease);
18874 let mut execution = resumed.claim_driver().await.unwrap();
18875 execution
18876 .driver
18877 .finish(&mut execution.runner.state)
18878 .await
18879 .unwrap();
18880 }
18881
18882 #[tokio::test]
18883 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18884 crate::run::pin_test_home();
18885 let mut runner = runner_at(RunStatus::Reviewing);
18886 let state = &mut runner.state;
18887 state.config.agents = vec![conductor()];
18888 state.driver_claimed = None;
18889 state.driver_pid = Some(std::process::id());
18890 state.driver_started_at = None;
18891 state.begin_pass();
18892 state.save().unwrap();
18893 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18894 std::fs::write(
18895 state.dir().join("driver.json"),
18896 serde_json::to_vec(&serde_json::json!({
18897 "pid": state.driver_pid, "started_at": null,
18898 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18899 }))
18900 .unwrap(),
18901 )
18902 .unwrap();
18903 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18904 drop(lease);
18905 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18906 let mut resumed = Runner::resume(&state.id).unwrap();
18907 assert!(!resumed.state.driver_exited);
18908 let mut execution = resumed
18909 .claim_driver()
18910 .await
18911 .expect("lease proves previous execution ended");
18912 assert_eq!(execution.runner.state.passes.len(), 2);
18913 execution
18914 .driver
18915 .finish(&mut execution.runner.state)
18916 .await
18917 .unwrap();
18918 }
18919
18920 #[tokio::test]
18921 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18922 crate::run::pin_test_home();
18923 let mut runner = runner_at(RunStatus::Reviewing);
18924 let mut json = serde_json::to_value(&runner.state).unwrap();
18925 json.as_object_mut().unwrap().remove("driver_claimed");
18926 runner.state = serde_json::from_value(json).unwrap();
18927 assert_eq!(runner.state.driver_claimed, None);
18928 runner.state.save().unwrap();
18929 let mut execution = runner
18930 .claim_driver()
18931 .await
18932 .expect("the execution lease, not a missing marker, excludes a second driver");
18933 assert_eq!(execution.runner.state.passes.len(), 1);
18934 execution
18935 .driver
18936 .finish(&mut execution.runner.state)
18937 .await
18938 .unwrap();
18939 }
18940
18941 #[tokio::test]
18942 async fn driver_admission_refuses_a_proven_live_driver() {
18943 crate::run::pin_test_home();
18944 let mut runner = runner_at(RunStatus::Reviewing);
18945 let pid = std::process::id();
18946 runner.state.driver_pid = Some(pid);
18947 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18948 assert!(runner.state.driver_started_at.is_some());
18949 runner.state.save().unwrap();
18950 let error = runner
18951 .claim_driver()
18952 .await
18953 .err()
18954 .expect("a live driver must refuse");
18955 assert!(error.to_string().contains("live driver"), "{error:#}");
18956 let saved = RunState::load(&runner.state.id).unwrap();
18957 assert_eq!(saved.driver_pid, Some(pid));
18958 assert_eq!(saved.status, RunStatus::Reviewing);
18959 assert!(saved.passes.is_empty());
18960 }
18961
18962 #[test]
18963 fn parking_resource_halt_persists_the_pass_once() {
18964 crate::run::pin_test_home();
18965 let mut runner = runner_at(RunStatus::Reviewing);
18966 runner.state.begin_pass();
18967 runner.state.resource_block = Some(crate::run::ResourceBlock {
18968 node: "fix".into(),
18969 kind: "shared build cache".into(),
18970 detail: "cancelled".into(),
18971 at: Timestamp::now(),
18972 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18973 });
18974 let cause = ParkCause::Upgrade {
18975 from: "old".into(),
18976 to: Some("new".into()),
18977 };
18978 runner.pause.park_for(cause.clone());
18979 runner.stop_for_resource().unwrap();
18982 assert!(runner.resource_halt().unwrap());
18983 assert!(runner.park_here().unwrap());
18984 let saved = RunState::load(&runner.state.id).unwrap();
18985 assert!(saved.parked);
18986 assert_eq!(saved.status, RunStatus::Reviewing);
18987 assert!(saved.passes[0].ended_at.is_some());
18988 assert_eq!(
18989 saved.passes[0].end,
18990 Some(crate::run::PassEnd::Parked {
18991 cause: cause.clone()
18992 })
18993 );
18994 assert_eq!(
18995 saved.resource_block.unwrap().cause,
18996 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18997 );
18998 assert_eq!(
18999 saved
19000 .events
19001 .iter()
19002 .filter(|event| event.node == "park")
19003 .count(),
19004 1
19005 );
19006 }
19007
19008 #[test]
19016 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19017 crate::run::pin_test_home();
19018 let mut runner = runner_at(RunStatus::Implementing);
19019 let shutdown = Pause::new();
19020 runner.on_pause(shutdown.clone());
19021 let interrupt = Pause::new();
19022 runner.watch_interrupt(interrupt.clone());
19023
19024 assert!(!runner.park_here().expect("park_here"));
19026 assert!(!runner.state.parked);
19027
19028 interrupt.park_because("test");
19030 assert!(!shutdown.parked());
19031 assert!(runner.park_here().expect("park_here"));
19032 }
19033
19034 #[tokio::test]
19048 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19049 crate::run::pin_test_home();
19050 let mut runner = runner_at(RunStatus::Implementing);
19051 let interrupt = Pause::new();
19052 runner.watch_interrupt(interrupt.clone());
19053
19054 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19055 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19056
19057 let node = async move {
19061 started_tx.send(()).expect("send started");
19062 finish_rx.await.expect("recv finish");
19063 "node finished"
19064 };
19065
19066 let interrupter = async move {
19067 started_rx.await.expect("recv started");
19068 interrupt.park_because("higher-priority task waiting");
19070 tokio::task::yield_now().await;
19074 finish_tx.send(()).expect("send finish");
19075 };
19076
19077 let (node_result, ()) = tokio::join!(node, interrupter);
19078 assert_eq!(
19079 node_result, "node finished",
19080 "the in-flight call ran to completion"
19081 );
19082
19083 assert!(runner.park_here().expect("park_here"));
19086 assert!(runner.state.parked);
19087 }
19088
19089 #[test]
19095 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19096 crate::run::pin_test_home();
19097 let mut runner = runner_at(RunStatus::Judging);
19098 runner.state.config.agents = vec![conductor()];
19102 runner.state.candidates = vec![Candidate {
19103 index: 0,
19104 label: 'A',
19105 agent: "alpha".to_owned(),
19106 branch: "magi/x/A".to_owned(),
19107 worktree: PathBuf::from("/nonexistent/worktree"),
19108 summary: "did the thing".to_owned(),
19109 stat: "1 file changed".to_owned(),
19110 files: 1,
19111 commits: 1,
19112 empty: false,
19113 failed: None,
19114 verified_noop: None,
19115 duration_ms: 1234,
19116 folded: false,
19117 }];
19118 let run_id = runner.state.id.clone();
19119
19120 let interrupt = Pause::new();
19121 runner.watch_interrupt(interrupt.clone());
19122 interrupt.park_because("task c3d4 asked to run first");
19123 assert!(runner.park_here().expect("park_here"));
19124
19125 let resumed = Runner::resume(&run_id).expect("resume");
19126 assert_eq!(resumed.state.candidates.len(), 1);
19127 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19128 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19129 assert_eq!(resumed.state.status, runner.state.status);
19130 assert!(
19131 resumed.state.parked,
19132 "still parked until `execute` actually walks the graph again"
19133 );
19134 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19135 }
19136
19137 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19139 let mut q = ask::Question::new(
19140 run.to_owned(),
19141 "implement".to_owned(),
19142 "impl-A".to_owned(),
19143 "Which storage backend should the cache use?".to_owned(),
19144 String::new(),
19145 vec!["SQLite".to_owned(), "Redis".to_owned()],
19146 );
19147 store.put(&mut q).unwrap();
19148 q
19149 }
19150
19151 #[test]
19152 fn a_failed_runs_open_question_is_abandoned() {
19153 ask_test_home();
19154 let store = ask::Questions::open();
19155 let mut runner = runner_at(RunStatus::Failed);
19156 let run = runner.state.id.clone();
19157 let q = ask_open_question(&store, &run);
19158
19159 runner.settle_questions();
19160
19161 let back = store.get(&q.id).unwrap();
19162 assert!(
19163 !back.status.open(),
19164 "the seat that asked died with the run; nobody is left to read an answer"
19165 );
19166 assert!(
19167 back.detail.contains(&run) && back.detail.contains("failed"),
19168 "the reason names what the run became, not just that it is gone: {}",
19169 back.detail
19170 );
19171 }
19172
19173 #[test]
19174 fn a_merged_runs_open_question_is_abandoned_too() {
19175 ask_test_home();
19176 let store = ask::Questions::open();
19177 for status in [RunStatus::Merged, RunStatus::Ready] {
19180 let mut runner = runner_at(status);
19181 let run = runner.state.id.clone();
19182 let q = ask_open_question(&store, &run);
19183
19184 runner.settle_questions();
19185
19186 let back = store.get(&q.id).unwrap();
19187 assert!(
19188 !back.status.open(),
19189 "{status:?} run's question must not outlive the run"
19190 );
19191 }
19192 }
19193
19194 #[test]
19195 fn a_still_resumable_runs_open_question_is_left_alone() {
19196 ask_test_home();
19197 let store = ask::Questions::open();
19198 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19204 let mut runner = runner_at(status);
19205 let run = runner.state.id.clone();
19206 let q = ask_open_question(&store, &run);
19207
19208 runner.settle_questions();
19209
19210 let back = store.get(&q.id).unwrap();
19211 assert!(
19212 back.status.open(),
19213 "{status:?} is still alive; the question must still be waiting"
19214 );
19215 }
19216 }
19217
19218 #[test]
19219 fn settle_questions_never_touches_an_already_answered_question() {
19220 ask_test_home();
19221 let store = ask::Questions::open();
19222 let mut runner = runner_at(RunStatus::Failed);
19223 let run = runner.state.id.clone();
19224 let mut q = ask_open_question(&store, &run);
19225 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19226 .unwrap();
19227 store.put(&mut q).unwrap();
19228
19229 runner.settle_questions();
19234 runner.settle_questions();
19235
19236 let back = store.get(&q.id).unwrap();
19237 assert_eq!(
19238 back.status,
19239 ask::QuestionStatus::Answered,
19240 "a real answer is a decision on record, never overwritten by a sweep"
19241 );
19242 }
19243
19244 #[tokio::test]
19255 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19256 crate::run::pin_test_home();
19257 let tmp = tempfile::tempdir().expect("tempdir");
19258 let repo = tmp.path().join("repo");
19259 std::fs::create_dir_all(&repo).unwrap();
19260 init_repo(&repo);
19261
19262 let mut config = Config::default();
19263 config.graph.worktree_root = Some(tmp.path().join("wt"));
19264
19265 let mut state = RunState::new(
19266 repo.clone(),
19267 "main".to_owned(),
19268 "deadbeef".to_owned(),
19269 "task".to_owned(),
19270 config,
19271 );
19272 let root = state.worktree_root();
19273 let wt_a = root.join("cand-A");
19274 let wt_b = root.join("cand-B");
19275 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19276 .await
19277 .expect("worktree A");
19278 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19279 .await
19280 .expect("worktree B");
19281
19282 state.candidates = vec![
19283 Candidate {
19284 index: 0,
19285 label: 'A',
19286 agent: "alpha".to_owned(),
19287 branch: "magi/x/A".to_owned(),
19288 worktree: wt_a.clone(),
19289 summary: String::new(),
19290 stat: String::new(),
19291 files: 0,
19292 commits: 0,
19293 empty: false,
19294 failed: None,
19295 verified_noop: None,
19296 duration_ms: 0,
19297 folded: false,
19298 },
19299 Candidate {
19300 index: 1,
19301 label: 'B',
19302 agent: "beta".to_owned(),
19303 branch: "magi/x/B".to_owned(),
19304 worktree: wt_b.clone(),
19305 summary: String::new(),
19306 stat: String::new(),
19307 files: 0,
19308 commits: 0,
19309 empty: false,
19310 failed: None,
19311 verified_noop: None,
19312 duration_ms: 0,
19313 folded: false,
19314 },
19315 ];
19316 state.tally = Some(Tally {
19317 first_choice: BTreeMap::from([('A', 1)]),
19318 borda: BTreeMap::new(),
19319 winner: 'A',
19320 rankings: 1,
19321 unanimous_initial: true,
19322 deliberated: false,
19323 changed_votes: 0,
19324 unanimous_final: true,
19325 tie_break: None,
19326 judges: 1,
19327 present: 1,
19328 quorum: 1,
19329 met_quorum: true,
19330 uncontested: None,
19331 });
19332 state.status = RunStatus::Ready;
19333
19334 fold_run(&mut state, false, &crate::run::home())
19335 .await
19336 .expect("fold_run");
19337
19338 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19339 assert!(
19340 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19341 "the unmerged winner's branch survives"
19342 );
19343 assert!(
19344 !state.candidates[0].folded,
19345 "the winner is not marked folded"
19346 );
19347
19348 assert!(!wt_b.exists(), "the loser's worktree is removed");
19349 assert!(
19350 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19351 "the loser's branch is removed"
19352 );
19353 assert!(state.candidates[1].folded, "the loser is marked folded");
19354 }
19355
19356 #[tokio::test]
19357 #[cfg(unix)]
19358 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19359 use std::os::fd::AsRawFd as _;
19360 use std::time::Duration;
19361
19362 let tmp = tempfile::tempdir().unwrap();
19363 let repo = tmp.path().join("repo");
19364 std::fs::create_dir_all(&repo).unwrap();
19365 init_repo(&repo);
19366 let common = repo.join(".git");
19367 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19368 let home = tmp.path().join("home");
19369 let mut config = Config::default();
19370 config.coordination = crate::config::test_coordination();
19371 config.graph.worktree_root = Some(tmp.path().join("wt"));
19372 config.graph.timeout_resource_wait = 1;
19373 let mut state = RunState::new(
19374 repo.clone(),
19375 "main".into(),
19376 "deadbeef".into(),
19377 "task".into(),
19378 config,
19379 );
19380 let root = state.worktree_root();
19381 let candidate = root.join("cand-A");
19382 let reviewer = root.join("review-1");
19383 let branch = state.branch_for('A');
19384 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19385 .await
19386 .unwrap();
19387 git::worktree_add_detached(&repo, &reviewer, "main")
19388 .await
19389 .unwrap();
19390 git::acquire_worktree_config(&repo).await.unwrap();
19391 state.enabled_worktree_config = true;
19392 state.status = RunStatus::Ready;
19393 state.candidates.push(Candidate {
19394 index: 0,
19395 label: 'A',
19396 agent: "implement".into(),
19397 branch: branch.clone(),
19398 worktree: candidate.clone(),
19399 summary: String::new(),
19400 stat: String::new(),
19401 files: 0,
19402 commits: 0,
19403 empty: false,
19404 failed: None,
19405 verified_noop: None,
19406 duration_ms: 0,
19407 folded: false,
19408 });
19409 state.save_under(&home).unwrap();
19410 let config_before = std::fs::read(common.join("config")).unwrap();
19411 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19412 .await
19413 .unwrap();
19414 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19417 assert_eq!(
19418 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19419 0
19420 );
19421 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19422 assert!(
19423 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19424 .await
19425 .is_err()
19426 );
19427 assert!(candidate.exists());
19428 assert!(reviewer.exists());
19429 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19430 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19431 assert_eq!(
19432 git::git(&repo, &["worktree", "list", "--porcelain"])
19433 .await
19434 .unwrap(),
19435 worktrees_before
19436 );
19437 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19438 .await
19439 .unwrap()
19440 .unwrap_err();
19441 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19442 drop(fold);
19443 assert!(candidate.exists() && reviewer.exists());
19444 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19445 assert_eq!(
19446 state.resource_block.as_ref().unwrap().kind,
19447 "repository coordination"
19448 );
19449 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19450 drop(external);
19451 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19452 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19453 .await
19454 .unwrap()
19455 .unwrap();
19456 drop(fold);
19457 assert!(removed.contains(&branch));
19458 assert!(!root.exists());
19459 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19460 assert!(state.candidates[0].folded);
19461 assert!(!state.enabled_worktree_config);
19462 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19463 let enabled = git::git_raw(
19464 &repo,
19465 &["config", "--local", "--get", "extensions.worktreeConfig"],
19466 )
19467 .await
19468 .unwrap();
19469 assert!(!enabled.ok(), "the last config reference was released");
19470 }
19471
19472 #[tokio::test]
19475 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19476 let tmp = tempfile::tempdir().expect("tempdir");
19477 let repo = tmp.path().join("repo");
19478 std::fs::create_dir_all(&repo).unwrap();
19479 init_repo(&repo);
19480 let home = tmp.path().join("home");
19481
19482 let mut config = Config::default();
19483 config.graph.worktree_root = Some(tmp.path().join("wt"));
19484 let mut state = RunState::new(
19485 repo.clone(),
19486 "main".to_owned(),
19487 "deadbeef".to_owned(),
19488 "task".to_owned(),
19489 config,
19490 );
19491 git::git(&repo, &["branch", "magi/x/A", "main"])
19493 .await
19494 .expect("branch");
19495 state.candidates = vec![Candidate {
19496 index: 0,
19497 label: 'A',
19498 agent: "alpha".to_owned(),
19499 branch: "magi/x/A".to_owned(),
19500 worktree: state.worktree_root().join("cand-A"),
19501 summary: String::new(),
19502 stat: String::new(),
19503 files: 0,
19504 commits: 0,
19505 empty: false,
19506 failed: None,
19507 verified_noop: None,
19508 duration_ms: 0,
19509 folded: true,
19510 }];
19511 state.released_to = Some("20260901-000000-new1".to_owned());
19512 state.released_branches = vec!["magi/x/A".to_owned()];
19513
19514 fold_run(&mut state, true, &home).await.expect("fold_run");
19515 state.status = RunStatus::Merged;
19516 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19517 state.merge = Some(crate::run::MergeOutcome {
19518 mode: MergeMode::Pr,
19519 ok: true,
19520 detail: "confirmed merged".into(),
19521 empty: false,
19522 });
19523 fold_run(&mut state, true, &home)
19524 .await
19525 .expect("merged fold");
19526
19527 assert!(
19528 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19529 "the handed-over branch survives a fold"
19530 );
19531 }
19532
19533 #[tokio::test]
19537 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19538 let tmp = tempfile::tempdir().expect("tempdir");
19539 let repo = tmp.path().join("repo");
19540 std::fs::create_dir_all(&repo).unwrap();
19541 init_repo(&repo);
19542 let run = |args: &[&str]| {
19543 let out = std::process::Command::new("git")
19544 .quiet()
19545 .args(args)
19546 .current_dir(&repo)
19547 .output()
19548 .expect("spawn git");
19549 assert!(out.status.success(), "git {args:?}");
19550 };
19551 run(&["branch", "magi/x/A"]);
19552 run(&["checkout", "-q", "-b", "magi/x/B"]);
19553 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19554 run(&["add", "-A"]);
19555 run(&["commit", "-q", "-m", "work"]);
19556 run(&["checkout", "-q", "main"]);
19557
19558 let probe = RunState::new(
19562 repo.clone(),
19563 "main".to_owned(),
19564 "deadbeef".to_owned(),
19565 "task".to_owned(),
19566 Config::default(),
19567 );
19568 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19569 let bare = tmp.path().join("origin.git");
19570 let out = std::process::Command::new("git")
19571 .quiet()
19572 .args(["init", "-q", "--bare"])
19573 .arg(&bare)
19574 .output()
19575 .expect("spawn git");
19576 assert!(out.status.success(), "git init --bare");
19577 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19578 run(&["push", "-q", "origin", "main"]);
19579
19580 let mut state = RunState::new(
19581 repo.clone(),
19582 "main".to_owned(),
19583 "deadbeef".to_owned(),
19584 "task".to_owned(),
19585 Config::default(),
19586 );
19587 state.seeds = vec![refs::Seed {
19588 token: "magi/27b2/A".to_owned(),
19589 kind: refs::SeedKind::Unresolved,
19590 sha: String::new(),
19591 branch: true,
19592 detail: "no branch or commit named magi/27b2/A".to_owned(),
19593 }];
19594
19595 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19596 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19597 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19598 let detail = empty_candidate_detail(&state, "main");
19599 assert!(detail.starts_with("empty candidate"), "{detail}");
19600 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19601 assert!(detail.contains("--start-from"), "{detail}");
19602 }
19603
19604 #[tokio::test]
19613 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19614 let tmp = tempfile::tempdir().expect("tempdir");
19615 let repo = tmp.path().join("repo");
19616 std::fs::create_dir_all(&repo).unwrap();
19617 init_repo(&repo);
19618
19619 let mut config = Config::default();
19620 config.merge.mode = MergeMode::Local;
19621
19622 let mut state = RunState::new(
19623 repo.clone(),
19624 "main".to_owned(),
19625 "deadbeef".to_owned(),
19626 "task".to_owned(),
19627 config,
19628 );
19629 state.candidates = vec![Candidate {
19630 index: 0,
19631 label: 'A',
19632 agent: "alpha".to_owned(),
19633 branch: "does-not-exist".to_owned(),
19634 worktree: repo.clone(),
19635 summary: String::new(),
19636 stat: String::new(),
19637 files: 0,
19638 commits: 0,
19639 empty: false,
19640 failed: None,
19641 verified_noop: None,
19642 duration_ms: 0,
19643 folded: false,
19644 }];
19645 state.tally = Some(Tally {
19646 first_choice: BTreeMap::from([('A', 1)]),
19647 borda: BTreeMap::new(),
19648 winner: 'A',
19649 rankings: 1,
19650 unanimous_initial: true,
19651 deliberated: false,
19652 changed_votes: 0,
19653 unanimous_final: true,
19654 tie_break: None,
19655 judges: 0,
19656 present: 0,
19657 quorum: 0,
19658 met_quorum: true,
19659 uncontested: Some("only candidate A produced a change".to_owned()),
19660 });
19661 state.reviews = vec![ReviewRound {
19662 round: 1,
19663 head: "deadbeef".to_owned(),
19664 verified_head: None,
19665 verified_at: None,
19666 reviews: Vec::new(),
19667 e2e: Vec::new(),
19668 fix: None,
19669 blocking: 0,
19670 answered: 0,
19671 expected: 0,
19672 clean: true,
19673 verify_retried: false,
19674 e2e_deferred: false,
19675 e2e_defer_reason: None,
19676 progressed: false,
19677 vote_split: false,
19678 reconsideration: Vec::new(),
19679 verdict: None,
19680 }];
19681 state.gate = vec![CommandOutcome {
19682 command: "test".to_owned(),
19683 code: Some(0),
19684 output_tail: String::new(),
19685 duration_ms: 0,
19686 resource_blocked: false,
19687 }];
19688 state.gate_ran = true;
19689 state.status = RunStatus::Ready;
19694 state.merge = Some(MergeOutcome {
19695 mode: MergeMode::Local,
19696 ok: false,
19697 detail: "already concluded".to_owned(),
19698 empty: false,
19699 });
19700
19701 let mut runner = Runner {
19702 state,
19703 roles: ResolvedRoles {
19704 implementers: Vec::new(),
19705 judges: Vec::new(),
19706 reviewers: Vec::new(),
19707 fixer: None,
19708 conductor: conductor(),
19709 implementer_roster: Vec::new(),
19710 judge_roster: Vec::new(),
19711 reviewer_roster: Vec::new(),
19712 },
19713 sem: Arc::new(Semaphore::new(1)),
19714 pause: Pause::new(),
19715 interrupt: Pause::new(),
19716 };
19717
19718 runner.merge().await.expect("merge");
19719
19720 assert_eq!(
19721 runner.state.status,
19722 RunStatus::Ready,
19723 "a concluded run's status must not change on reentry"
19724 );
19725 assert_eq!(
19726 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19727 Some("already concluded"),
19728 "merge must not run again once the node already recorded an outcome"
19729 );
19730 }
19731
19732 #[tokio::test]
19741 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19742 let tmp = tempfile::tempdir().expect("tempdir");
19743 let repo = tmp.path().join("repo");
19744 std::fs::create_dir_all(&repo).unwrap();
19745 init_repo(&repo);
19746
19747 let mut config = Config::default();
19748 config.merge.mode = MergeMode::Local;
19749
19750 let mut state = RunState::new(
19751 repo.clone(),
19752 "main".to_owned(),
19753 "deadbeef".to_owned(),
19754 "task".to_owned(),
19755 config,
19756 );
19757 state.candidates = vec![Candidate {
19758 index: 0,
19759 label: 'A',
19760 agent: "alpha".to_owned(),
19761 branch: "does-not-exist".to_owned(),
19762 worktree: repo.clone(),
19763 summary: String::new(),
19764 stat: String::new(),
19765 files: 0,
19766 commits: 0,
19767 empty: false,
19768 failed: None,
19769 verified_noop: None,
19770 duration_ms: 0,
19771 folded: false,
19772 }];
19773 state.tally = Some(Tally {
19774 first_choice: BTreeMap::from([('A', 1)]),
19775 borda: BTreeMap::new(),
19776 winner: 'A',
19777 rankings: 1,
19778 unanimous_initial: true,
19779 deliberated: false,
19780 changed_votes: 0,
19781 unanimous_final: true,
19782 tie_break: None,
19783 judges: 0,
19784 present: 0,
19785 quorum: 0,
19786 met_quorum: true,
19787 uncontested: Some("only candidate A produced a change".to_owned()),
19788 });
19789 state.reviews = vec![ReviewRound {
19790 round: 1,
19791 head: "deadbeef".to_owned(),
19792 verified_head: None,
19793 verified_at: None,
19794 reviews: Vec::new(),
19795 e2e: Vec::new(),
19796 fix: None,
19797 blocking: 0,
19798 answered: 0,
19799 expected: 0,
19800 clean: true,
19801 verify_retried: false,
19802 e2e_deferred: false,
19803 e2e_defer_reason: None,
19804 progressed: false,
19805 vote_split: false,
19806 reconsideration: Vec::new(),
19807 verdict: None,
19808 }];
19809 state.gate = Vec::new();
19811 state.gate_ran = false;
19812 state.status = RunStatus::Gating;
19813
19814 let mut runner = Runner {
19815 state,
19816 roles: ResolvedRoles {
19817 implementers: Vec::new(),
19818 judges: Vec::new(),
19819 reviewers: Vec::new(),
19820 fixer: None,
19821 conductor: conductor(),
19822 implementer_roster: Vec::new(),
19823 judge_roster: Vec::new(),
19824 reviewer_roster: Vec::new(),
19825 },
19826 sem: Arc::new(Semaphore::new(1)),
19827 pause: Pause::new(),
19828 interrupt: Pause::new(),
19829 };
19830
19831 runner.merge().await.expect("merge");
19832
19833 assert!(
19834 runner.state.merge.is_none(),
19835 "an empty gate must never be read as a passing one: {:?}",
19836 runner.state.merge
19837 );
19838 }
19839
19840 #[tokio::test]
19847 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19848 let tmp = tempfile::tempdir().expect("tempdir");
19849 let repo = tmp.path().join("repo");
19850 std::fs::create_dir_all(&repo).unwrap();
19851 init_repo(&repo);
19852
19853 let config = Config::default();
19855
19856 let mut state = RunState::new(
19857 repo.clone(),
19858 "main".to_owned(),
19859 "deadbeef".to_owned(),
19860 "task".to_owned(),
19861 config,
19862 );
19863 state.candidates = vec![Candidate {
19864 index: 0,
19865 label: 'A',
19866 agent: "alpha".to_owned(),
19867 branch: "does-not-exist".to_owned(),
19868 worktree: repo.clone(),
19869 summary: String::new(),
19870 stat: String::new(),
19871 files: 0,
19872 commits: 0,
19873 empty: false,
19874 failed: None,
19875 verified_noop: None,
19876 duration_ms: 0,
19877 folded: false,
19878 }];
19879 state.tally = Some(Tally {
19880 first_choice: BTreeMap::from([('A', 1)]),
19881 borda: BTreeMap::new(),
19882 winner: 'A',
19883 rankings: 1,
19884 unanimous_initial: true,
19885 deliberated: false,
19886 changed_votes: 0,
19887 unanimous_final: true,
19888 tie_break: None,
19889 judges: 0,
19890 present: 0,
19891 quorum: 0,
19892 met_quorum: true,
19893 uncontested: Some("only candidate A produced a change".to_owned()),
19894 });
19895 state.reviews = vec![ReviewRound {
19896 round: 1,
19897 head: "deadbeef".to_owned(),
19898 verified_head: None,
19899 verified_at: None,
19900 reviews: Vec::new(),
19901 e2e: Vec::new(),
19902 fix: None,
19903 blocking: 0,
19904 answered: 0,
19905 expected: 0,
19906 clean: true,
19907 verify_retried: false,
19908 e2e_deferred: false,
19909 e2e_defer_reason: None,
19910 progressed: false,
19911 vote_split: false,
19912 reconsideration: Vec::new(),
19913 verdict: None,
19914 }];
19915
19916 let mut runner = Runner {
19917 state,
19918 roles: ResolvedRoles {
19919 implementers: Vec::new(),
19920 judges: Vec::new(),
19921 reviewers: Vec::new(),
19922 fixer: None,
19923 conductor: conductor(),
19924 implementer_roster: Vec::new(),
19925 judge_roster: Vec::new(),
19926 reviewer_roster: Vec::new(),
19927 },
19928 sem: Arc::new(Semaphore::new(1)),
19929 pause: Pause::new(),
19930 interrupt: Pause::new(),
19931 };
19932
19933 runner.gate().await.expect("gate");
19934 assert!(
19935 runner.state.gate_ran,
19936 "zero configured commands is still a real attempt, not an unrun gate"
19937 );
19938 assert!(runner.state.gate.is_empty());
19939 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19940 assert_ne!(
19941 runner.state.status,
19942 RunStatus::Blocked,
19943 "a gate with nothing to check must not read as failed"
19944 );
19945
19946 runner.merge().await.expect("merge");
19947 assert_eq!(
19948 runner.state.status,
19949 RunStatus::Ready,
19950 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19951 );
19952 }
19953
19954 #[tokio::test]
19965 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19966 crate::run::pin_test_home();
19967 let home = crate::run::home();
19968
19969 let tmp = tempfile::tempdir().expect("tempdir");
19970 let repo = tmp.path().join("repo");
19971 std::fs::create_dir_all(&repo).unwrap();
19972 init_repo(&repo);
19973 let cache_dir = tmp.path().join("target");
19976
19977 let mut config = Config::default();
19978 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19979 config.graph.timeout_verify = Some(2);
19983 config.graph.timeout_resource_wait = 0;
19984
19985 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19986 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19987 .expect("no io error acquiring directly")
19988 {
19989 crate::cache::AcquireOutcome::Acquired(g) => g,
19990 crate::cache::AcquireOutcome::Busy(b) => {
19991 panic!("expected the direct acquire to win the lease first: {b:?}")
19992 }
19993 };
19994
19995 let mut state = RunState::new(
19996 repo.clone(),
19997 "main".to_owned(),
19998 "deadbeef".to_owned(),
19999 "task".to_owned(),
20000 config,
20001 );
20002 state.candidates = vec![Candidate {
20003 index: 0,
20004 label: 'A',
20005 agent: "alpha".to_owned(),
20006 branch: "does-not-exist".to_owned(),
20007 worktree: repo.clone(),
20008 summary: String::new(),
20009 stat: String::new(),
20010 files: 0,
20011 commits: 0,
20012 empty: false,
20013 failed: None,
20014 verified_noop: None,
20015 duration_ms: 0,
20016 folded: false,
20017 }];
20018 state.tally = Some(Tally {
20019 first_choice: BTreeMap::from([('A', 1)]),
20020 borda: BTreeMap::new(),
20021 winner: 'A',
20022 rankings: 1,
20023 unanimous_initial: true,
20024 deliberated: false,
20025 changed_votes: 0,
20026 unanimous_final: true,
20027 tie_break: None,
20028 judges: 0,
20029 present: 0,
20030 quorum: 0,
20031 met_quorum: true,
20032 uncontested: Some("only candidate A produced a change".to_owned()),
20033 });
20034 state.reviews = vec![ReviewRound {
20035 round: 1,
20036 head: "deadbeef".to_owned(),
20037 verified_head: None,
20038 verified_at: None,
20039 reviews: Vec::new(),
20040 e2e: Vec::new(),
20041 fix: None,
20042 blocking: 0,
20043 answered: 0,
20044 expected: 0,
20045 clean: true,
20046 verify_retried: false,
20047 e2e_deferred: false,
20048 e2e_defer_reason: None,
20049 progressed: false,
20050 vote_split: false,
20051 reconsideration: Vec::new(),
20052 verdict: None,
20053 }];
20054
20055 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20056 state.reviews[0].e2e = vec![CommandOutcome {
20057 command: state.config.verify.e2e[0].clone(),
20058 code: Some(0),
20059 output_tail: String::new(),
20060 duration_ms: 0,
20061 resource_blocked: false,
20062 }];
20063 let mut runner = Runner {
20064 state,
20065 roles: ResolvedRoles {
20066 implementers: Vec::new(),
20067 judges: Vec::new(),
20068 reviewers: Vec::new(),
20069 fixer: None,
20070 conductor: conductor(),
20071 implementer_roster: Vec::new(),
20072 judge_roster: Vec::new(),
20073 reviewer_roster: Vec::new(),
20074 },
20075 sem: Arc::new(Semaphore::new(1)),
20076 pause: Pause::new(),
20077 interrupt: Pause::new(),
20078 };
20079
20080 runner.gate().await.expect("gate");
20081 assert!(runner.state.resource_block.is_none());
20082 assert!(
20083 !runner
20084 .state
20085 .events
20086 .iter()
20087 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20088 "a gate with nothing to run must never request the held cache lease"
20089 );
20090 assert!(
20091 runner.state.gate_ran,
20092 "zero commands is still a real, immediate attempt"
20093 );
20094 assert!(runner.state.gate.is_empty());
20095 assert_ne!(
20096 runner.state.status,
20097 RunStatus::Blocked,
20098 "must not read as resource-blocked on a lease it never asked for"
20099 );
20100 }
20101
20102 #[cfg(unix)]
20105 #[tokio::test]
20106 async fn verification_is_standalone_inside_a_live_worker() {
20107 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20108 if std::env::var_os(CHILD).is_none() {
20109 let home = tempfile::tempdir().unwrap();
20110 let generation = crate::worker::pin_current(home.path()).unwrap();
20111 let output = std::process::Command::new(std::env::current_exe().unwrap())
20112 .args([
20113 "--exact",
20114 "graph::tests::verification_is_standalone_inside_a_live_worker",
20115 "--nocapture",
20116 ])
20117 .env(CHILD, "1")
20118 .env("MAGI_HOME", home.path())
20119 .env("MAGI_WORKER_HOME", home.path())
20120 .env("MAGI_WORKER_ID", "abcdef")
20121 .env("MAGI_WORKER_GENERATION", &generation.id)
20122 .env("MAGI_WORKER_EXE", &generation.executable)
20123 .env("MAGI_RUN", "verification-parent")
20124 .env("MAGI_NODE", "gate")
20125 .env("MAGI_SEAT", "worker-fixture")
20126 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20127 .env("MAGI_NO_AUTOUPDATE", "0")
20128 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20129 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20130 .env(
20131 "VERIFY_EXPECT_TOOLCHAIN",
20132 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20133 )
20134 .env(
20135 "VERIFY_EXPECT_CACHE",
20136 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20137 )
20138 .output()
20139 .unwrap();
20140 assert!(
20141 output.status.success()
20142 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20143 "{}\n{}",
20144 String::from_utf8_lossy(&output.stdout),
20145 String::from_utf8_lossy(&output.stderr)
20146 );
20147 return;
20148 }
20149 let home = crate::run::home();
20150 let directory = home.join("workers/jobs/abcdef");
20151 std::fs::create_dir_all(&directory).unwrap();
20152 let job = crate::worker::Job {
20153 id: "abcdef".into(),
20154 sequence: 1,
20155 kind: "task".into(),
20156 key: "task:verification".into(),
20157 payload: serde_json::json!({}),
20158 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20159 version: Some(env!("CARGO_PKG_VERSION").into()),
20160 status: crate::worker::Status::Running,
20161 result: None,
20162 pid: Some(std::process::id()),
20163 created_at: 0,
20164 finished_at: None,
20165 };
20166 std::fs::write(
20167 directory.join("job.json"),
20168 serde_json::to_vec(&job).unwrap(),
20169 )
20170 .unwrap();
20171 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20172 let assert_worker = || {
20173 assert_eq!(
20174 crate::worker::current_job(&home).unwrap().unwrap().id,
20175 job.id
20176 );
20177 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20178 assert_eq!(
20179 crate::worker::helper_executable(&home).unwrap(),
20180 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20181 );
20182 assert!(
20183 crate::updater::Checker::new(&crate::config::Update {
20184 mode: crate::config::UpdateMode::Notify,
20185 interval: None,
20186 auto_restart: false,
20187 })
20188 .is_none(),
20189 "the driver remains pinned, even while tests are standalone"
20190 );
20191 };
20192 assert_worker();
20193 let repo = home.join("repo");
20194 std::fs::create_dir_all(&repo).unwrap();
20195 init_repo(&repo);
20196 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20199 test -z "${MAGI_WORKER_GENERATION+x}" &&
20200 test -z "${MAGI_WORKER_HOME+x}" &&
20201 test -z "${MAGI_WORKER_EXE+x}" &&
20202 test "$MAGI_HOME/repo" -ef . &&
20203 test "$MAGI_RUN" = verification-parent &&
20204 test "$MAGI_NODE" = gate &&
20205 test "$MAGI_SEAT" = worker-fixture &&
20206 test -n "$MAGI_PROCESS_OWNER" &&
20207 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20208 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20209 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20210 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20211 assert!(
20213 !std::process::Command::new("sh")
20214 .args(["-c", probe])
20215 .current_dir(&repo)
20216 .status()
20217 .unwrap()
20218 .success()
20219 );
20220 let unrelated_home = tempfile::tempdir().unwrap();
20221 assert!(
20222 crate::worker::current_job(unrelated_home.path()).is_err(),
20223 "an inherited worker id cannot be resolved in a unit fixture's home"
20224 );
20225 let mut runner = runner_at(RunStatus::Gating);
20226 runner.state.repo = repo.clone();
20227 runner.state.config.disk.min_free_bytes = 0;
20228 runner.state.config.coordination = crate::config::test_coordination();
20229 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20230 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20231 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20232 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20233 let winner = Candidate {
20234 index: 0,
20235 label: 'A',
20236 agent: "fixture".into(),
20237 branch: "main".into(),
20238 worktree: repo.clone(),
20239 summary: String::new(),
20240 stat: String::new(),
20241 files: 0,
20242 commits: 0,
20243 empty: false,
20244 failed: None,
20245 verified_noop: None,
20246 duration_ms: 0,
20247 folded: false,
20248 };
20249 let (e2e, retried, pids) = run_e2e_with_retry(
20250 &mut runner.state,
20251 &["sh".into(), "-c".into()],
20252 &[format!("{probe} && printf initial")],
20253 &repo,
20254 Duration::from_secs(10),
20255 "initial",
20256 )
20257 .await;
20258 assert!(!retried && pids.is_empty());
20259 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20260 assert_eq!(e2e[0].output_tail, "initial");
20261 let gate = runner.run_gate(&winner).await.unwrap();
20262 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20263 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20266 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20267 passed: false,
20268 outcomes: Vec::new(),
20269 });
20270 runner.state.save().unwrap();
20271 let mut resumed = RunState::load(&runner.state.id).unwrap();
20272 let wait = crate::repo_lock::WaitPolicy::default();
20273 assert!(matches!(
20274 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20275 .await
20276 .unwrap(),
20277 GateFix::Retry
20278 ));
20279 let saved = RunState::load(&resumed.id).unwrap();
20280 let repair = saved.gate_repair_e2e.unwrap();
20281 assert!(repair.passed);
20282 assert_eq!(repair.outcomes.len(), 1);
20283 assert!(repair.outcomes[0].ok());
20284 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20285 .await
20286 .unwrap();
20287 assert_eq!(landing.len(), 1);
20288 assert!(landing[0].ok(), "{landing:?}");
20289 assert_eq!(
20290 std::fs::read_to_string(repo.join("phases")).unwrap(),
20291 "pre\ngate\ne2e\npre\ngate\n"
20292 );
20293 assert_worker();
20294 }
20295
20296 #[cfg(unix)]
20297 #[tokio::test]
20298 async fn verification_keeps_the_panic_and_the_full_output() {
20299 crate::run::pin_test_home();
20300 let tmp = tempfile::tempdir().unwrap();
20301 let mut config = Config::default();
20302 config.disk.min_free_bytes = 0;
20303 let mut state = RunState::new(
20304 tmp.path().to_owned(),
20305 "main".into(),
20306 "base".into(),
20307 "verification evidence".into(),
20308 config,
20309 );
20310 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20311 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20312 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20313 let mut names = Vec::new();
20314 for _ in 0..2 {
20315 let (outcomes, _) = run_commands(
20316 &mut state,
20317 "gate",
20318 "gate",
20319 0,
20320 &["sh".into(), "-c".into()],
20321 &[script.into()],
20322 tmp.path(),
20323 Duration::from_secs(60),
20324 )
20325 .await;
20326 let tail = &outcomes[0].output_tail;
20327 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20328 assert!(tail.contains("boom assertion"));
20329 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20330 let line = tail.lines().next().unwrap();
20331 names.push(line.to_owned());
20332 }
20333 assert_ne!(names[0], names[1], "the second run must not overwrite");
20334 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20335 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20336 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20337
20338 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20340 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20341 let (outcomes, _) = run_commands(
20342 &mut state,
20343 "gate",
20344 "gate",
20345 0,
20346 &["sh".into(), "-c".into()],
20347 &["echo failure-output; exit 7".into()],
20348 tmp.path(),
20349 Duration::from_secs(10),
20350 )
20351 .await;
20352 assert_eq!(outcomes[0].code, Some(7));
20353 assert!(state.events.iter().any(|e| {
20354 e.node == "gate" && e.message.contains("could not save the full gate output:")
20355 }));
20356 }
20357
20358 #[cfg(unix)]
20359 #[tokio::test]
20360 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20361 crate::run::pin_test_home();
20362 let tmp = tempfile::tempdir().unwrap();
20363 let mut config = Config::default();
20364 config.disk.min_free_bytes = 0;
20365 let mut state = RunState::new(
20366 tmp.path().to_owned(),
20367 "main".into(),
20368 "base".into(),
20369 "verification evidence".into(),
20370 config,
20371 );
20372 let (outcomes, pids) = run_commands(
20373 &mut state,
20374 "gate",
20375 "gate",
20376 1,
20377 &["sh".into(), "-c".into()],
20378 &[
20379 "printf before-timeout; exec sleep 30".into(),
20380 "kill -TERM $$".into(),
20381 "exit 7".into(),
20382 "printf success".into(),
20383 ],
20384 tmp.path(),
20385 Duration::from_secs(1),
20386 )
20387 .await;
20388 wait_for_timed_out_children_to_die(&pids).await;
20389 assert_eq!(
20390 pids.len(),
20391 1,
20392 "unleased timeout retains its death-check PID"
20393 );
20394 assert_eq!(outcomes.len(), 4);
20395 assert_eq!(outcomes[0].code, None);
20396 assert!(outcomes[0].output_tail.contains("before-timeout"));
20397 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20398 assert_eq!(outcomes[1].code, None);
20399 assert!(outcomes[1].output_tail.contains("command terminated:"));
20400 assert!(!outcomes[1].output_tail.contains("timed out"));
20401 assert_eq!(outcomes[2].code, Some(7));
20402 assert!(!outcomes[2].output_tail.contains("timed out"));
20403 assert_eq!(outcomes[3].code, Some(0));
20404 assert_eq!(outcomes[3].output_tail, "success");
20405 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20406 }
20407
20408 #[tokio::test]
20418 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20419 crate::run::pin_test_home();
20420
20421 let tmp = tempfile::tempdir().expect("tempdir");
20422 let repo = tmp.path().join("repo");
20423 std::fs::create_dir_all(&repo).unwrap();
20424 init_repo(&repo);
20425
20426 let mut config = Config::default();
20427 config.disk.min_free_bytes = 0;
20428 config.verify.gate = vec![
20429 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20430 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20431 .to_owned(),
20432 ];
20433
20434 let mut state = RunState::new(
20435 repo.clone(),
20436 "main".to_owned(),
20437 "deadbeef".to_owned(),
20438 "task".to_owned(),
20439 config,
20440 );
20441 let run_id = state.id.clone();
20442 state.candidates = vec![Candidate {
20443 index: 0,
20444 label: 'A',
20445 agent: "alpha".to_owned(),
20446 branch: "does-not-exist".to_owned(),
20447 worktree: repo.clone(),
20448 summary: String::new(),
20449 stat: String::new(),
20450 files: 0,
20451 commits: 0,
20452 empty: false,
20453 failed: None,
20454 verified_noop: None,
20455 duration_ms: 0,
20456 folded: false,
20457 }];
20458 state.tally = Some(Tally {
20459 first_choice: BTreeMap::from([('A', 1)]),
20460 borda: BTreeMap::new(),
20461 winner: 'A',
20462 rankings: 1,
20463 unanimous_initial: true,
20464 deliberated: false,
20465 changed_votes: 0,
20466 unanimous_final: true,
20467 tie_break: None,
20468 judges: 0,
20469 present: 0,
20470 quorum: 0,
20471 met_quorum: true,
20472 uncontested: Some("only candidate A produced a change".to_owned()),
20473 });
20474 state.reviews = vec![ReviewRound {
20475 round: 1,
20476 head: "deadbeef".to_owned(),
20477 verified_head: None,
20478 verified_at: None,
20479 reviews: Vec::new(),
20480 e2e: Vec::new(),
20481 fix: None,
20482 blocking: 0,
20483 answered: 0,
20484 expected: 0,
20485 clean: true,
20486 verify_retried: false,
20487 e2e_deferred: false,
20488 e2e_defer_reason: None,
20489 progressed: false,
20490 vote_split: false,
20491 reconsideration: Vec::new(),
20492 verdict: None,
20493 }];
20494
20495 let mut runner = Runner {
20496 state,
20497 roles: ResolvedRoles {
20498 implementers: Vec::new(),
20499 judges: Vec::new(),
20500 reviewers: Vec::new(),
20501 fixer: None,
20502 conductor: conductor(),
20503 implementer_roster: Vec::new(),
20504 judge_roster: Vec::new(),
20505 reviewer_roster: Vec::new(),
20506 },
20507 sem: Arc::new(Semaphore::new(1)),
20508 pause: Pause::new(),
20509 interrupt: Pause::new(),
20510 };
20511
20512 let started_marker = repo.join("started.marker");
20513 let release_marker = repo.join("release.marker");
20514 let poller = tokio::spawn(async move {
20515 for _ in 0..100 {
20520 if started_marker.exists()
20521 && let Ok(s) = crate::run::RunState::load(&run_id)
20522 && let Some(a) = s.active.get("gate")
20523 {
20524 std::fs::write(&release_marker, b"go").expect("release marker");
20525 return Some(a.clone());
20526 }
20527 tokio::time::sleep(Duration::from_millis(50)).await;
20528 }
20529 None
20530 });
20531
20532 runner.gate().await.expect("gate");
20533 let captured = poller.await.expect("poller task");
20534 let captured = captured.expect(
20535 "the poller never saw a `gate` task entry in run.json while the command was \
20536 still blocked on its own release marker",
20537 );
20538
20539 assert_eq!(captured.task.as_deref(), Some("gate"));
20540 assert_eq!(captured.node, "gate");
20541 assert_eq!(captured.index, Some(1));
20542 assert_eq!(captured.total, Some(1));
20543 assert!(
20544 captured
20545 .command
20546 .as_deref()
20547 .is_some_and(|c| c.contains("started.marker")),
20548 "{captured:?}"
20549 );
20550
20551 assert!(
20552 runner.state.active.is_empty(),
20553 "the entry must be cleared once the command actually finished: {:?}",
20554 runner.state.active
20555 );
20556 assert!(runner.state.gate_ran);
20557 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20558 }
20559
20560 #[tokio::test]
20563 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20564 use crate::verdict::{Finding, ReviewVote, Severity};
20565 crate::run::pin_test_home();
20566 let tmp = tempfile::tempdir().expect("tempdir");
20567 let repo = tmp.path().join("repo");
20568 std::fs::create_dir_all(&repo).unwrap();
20569 init_repo(&repo);
20570
20571 for (severity, vote, expect) in [
20572 (Severity::Major, ReviewVote::Reject, true),
20573 (Severity::Minor, ReviewVote::Reject, false),
20574 (Severity::Major, ReviewVote::Approve, false),
20575 ] {
20576 let mut round = review_round(false, 1, 1, 1, false, true);
20577 round.reviews = vec![ReviewRecord {
20578 reviewer: 1,
20579 agent: "alpha".to_owned(),
20580 summary: String::new(),
20581 findings: vec![Finding {
20582 id: "R1-1-1".to_owned(),
20583 severity,
20584 file: None,
20585 line: None,
20586 title: "t".to_owned(),
20587 detail: String::new(),
20588 }],
20589 vote: Some(vote),
20590 failed: None,
20591 duration_ms: 0,
20592 attempts: 0,
20593 }];
20594 let mut state = RunState::new(
20595 repo.clone(),
20596 "main".to_owned(),
20597 "deadbeef".to_owned(),
20598 "task".to_owned(),
20599 Config::default(),
20600 );
20601 state.reviews = vec![round];
20602 let mut runner = Runner {
20603 state,
20604 roles: ResolvedRoles {
20605 implementers: Vec::new(),
20606 judges: Vec::new(),
20607 reviewers: Vec::new(),
20608 fixer: None,
20609 conductor: conductor(),
20610 implementer_roster: Vec::new(),
20611 judge_roster: Vec::new(),
20612 reviewer_roster: Vec::new(),
20613 },
20614 sem: Arc::new(Semaphore::new(1)),
20615 pause: Pause::new(),
20616 interrupt: Pause::new(),
20617 };
20618 let shell = runner.state.config.shell();
20619 runner
20620 .stop_reviewing("round budget spent", &shell, &repo)
20621 .await
20622 .expect("stop_reviewing");
20623 assert_eq!(runner.state.status, RunStatus::Gating);
20624 assert_eq!(
20625 runner.state.contested_handoff.is_some(),
20626 expect,
20627 "{severity:?} + {vote:?}"
20628 );
20629 }
20630 }
20631
20632 #[tokio::test]
20645 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20646 crate::run::pin_test_home();
20647 let home = crate::run::home();
20648
20649 let tmp = tempfile::tempdir().expect("tempdir");
20650 let repo = tmp.path().join("repo");
20651 std::fs::create_dir_all(&repo).unwrap();
20652 init_repo(&repo);
20653 let head = crate::git::rev_parse(&repo, "HEAD")
20654 .await
20655 .expect("rev-parse");
20656 let cache_dir = tmp.path().join("target");
20659
20660 let mut config = Config::default();
20661 config.verify.e2e = vec![format!(
20662 "CARGO_TARGET_DIR='{}' test -f README.md",
20663 cache_dir.display()
20664 )];
20665 config.graph.review_rounds = 1;
20666 config.graph.timeout_verify = Some(2);
20669 config.graph.timeout_resource_wait = 2;
20670
20671 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20672 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20673 .expect("no io error acquiring directly")
20674 {
20675 crate::cache::AcquireOutcome::Acquired(g) => g,
20676 crate::cache::AcquireOutcome::Busy(b) => {
20677 panic!("expected the direct acquire to win the lease first: {b:?}")
20678 }
20679 };
20680
20681 let mut state = RunState::new(
20682 repo.clone(),
20683 "main".to_owned(),
20684 head.clone(),
20685 "task".to_owned(),
20686 config,
20687 );
20688 state.candidates = vec![Candidate {
20689 index: 0,
20690 label: 'A',
20691 agent: "alpha".to_owned(),
20692 branch: "does-not-exist".to_owned(),
20693 worktree: repo.clone(),
20694 summary: String::new(),
20695 stat: String::new(),
20696 files: 0,
20697 commits: 0,
20698 empty: false,
20699 failed: None,
20700 verified_noop: None,
20701 duration_ms: 0,
20702 folded: false,
20703 }];
20704 state.tally = Some(Tally {
20705 first_choice: BTreeMap::from([('A', 1)]),
20706 borda: BTreeMap::new(),
20707 winner: 'A',
20708 rankings: 1,
20709 unanimous_initial: true,
20710 deliberated: false,
20711 changed_votes: 0,
20712 unanimous_final: true,
20713 tie_break: None,
20714 judges: 0,
20715 present: 0,
20716 quorum: 0,
20717 met_quorum: true,
20718 uncontested: Some("only candidate A produced a change".to_owned()),
20719 });
20720 state.reviews = vec![ReviewRound {
20724 round: 1,
20725 head: head.clone(),
20726 verified_head: None,
20727 verified_at: None,
20728 reviews: Vec::new(),
20729 e2e: Vec::new(),
20730 fix: None,
20731 blocking: 1,
20732 answered: 1,
20733 expected: 1,
20734 clean: false,
20735 verify_retried: false,
20736 e2e_deferred: true,
20737 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20738 progressed: false,
20739 vote_split: false,
20740 reconsideration: Vec::new(),
20741 verdict: None,
20742 }];
20743
20744 let mut runner = Runner {
20745 state,
20746 roles: ResolvedRoles {
20747 implementers: Vec::new(),
20748 judges: Vec::new(),
20749 reviewers: Vec::new(),
20750 fixer: None,
20751 conductor: conductor(),
20752 implementer_roster: Vec::new(),
20753 judge_roster: Vec::new(),
20754 reviewer_roster: Vec::new(),
20755 },
20756 sem: Arc::new(Semaphore::new(1)),
20757 pause: Pause::new(),
20758 interrupt: Pause::new(),
20759 };
20760
20761 let shell = runner.state.config.shell();
20762 runner
20763 .stop_reviewing("round budget spent", &shell, &repo)
20764 .await
20765 .expect("stop_reviewing");
20766
20767 let last = runner.state.reviews.last().expect("round record");
20768 assert_eq!(
20769 last.e2e_status(),
20770 E2eStatus::ResourceBlocked,
20771 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20772 failed: {last:?}"
20773 );
20774 assert_eq!(
20775 last.verified_head.as_deref(),
20776 Some(head.as_str()),
20777 "which commit this attempt targeted is known even though nothing finished checking \
20778 it"
20779 );
20780 let first_attempt_at = last
20781 .verified_at
20782 .expect("when this attempt ran is known too");
20783 assert_ne!(
20784 runner.state.status,
20785 RunStatus::Blocked,
20786 "contention is evidence about the machine, not the patch — it must not settle the \
20787 run as blocked: {:?}",
20788 runner.state.status
20789 );
20790 assert!(
20791 !runner
20792 .state
20793 .events
20794 .iter()
20795 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20796 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20797 runner.state.events
20798 );
20799
20800 runner
20805 .stop_reviewing("round budget spent", &shell, &repo)
20806 .await
20807 .expect("stop_reviewing retry");
20808 assert_eq!(
20809 runner.state.reviews.len(),
20810 1,
20811 "no new round was started: {:?}",
20812 runner.state.reviews
20813 );
20814 let last = runner.state.reviews.last().expect("round record");
20815 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20816 assert!(
20817 last.verified_at.expect("still known") > first_attempt_at,
20818 "a second reentry must be a fresh attempt, not a stale copy of the first"
20819 );
20820 assert_ne!(runner.state.status, RunStatus::Blocked);
20821
20822 held.release();
20823 }
20824
20825 #[cfg(unix)]
20826 #[tokio::test]
20827 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20828 ask_test_home();
20829 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20830 let tmp = tempfile::tempdir().unwrap();
20831 let repo = tmp.path().join("repo");
20832 std::fs::create_dir(&repo).unwrap();
20833 init_repo(&repo);
20834 let run_git = |args: &[&str]| {
20835 let out = std::process::Command::new("git")
20836 .args(args)
20837 .current_dir(&repo)
20838 .output()
20839 .unwrap();
20840 assert!(
20841 out.status.success(),
20842 "{args:?}: {}",
20843 String::from_utf8_lossy(&out.stderr)
20844 );
20845 };
20846 let origin = tmp.path().join("origin.git");
20847 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20848 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20849 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20850 run_git(&["add", "spec.rs"]);
20851 run_git(&["commit", "-m", "initial spec"]);
20852 run_git(&["push", "origin", "main"]);
20853 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20854 run_git(&["checkout", "-b", "fix/review"]);
20855 std::fs::write(
20856 repo.join("check.rs"),
20857 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20858 )
20859 .unwrap();
20860 run_git(&["add", "check.rs"]);
20861 run_git(&["commit", "-m", "use spec"]);
20862 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20863 run_git(&["checkout", "main"]);
20864 std::fs::write(
20865 repo.join("spec.rs"),
20866 "struct AgentSpec { max_turn_secs: u64 }\n",
20867 )
20868 .unwrap();
20869 run_git(&["add", "spec.rs"]);
20870 run_git(&["commit", "-m", "extend spec"]);
20871 run_git(&["push", "origin", "main"]);
20872 run_git(&["checkout", "fix/review"]);
20873
20874 let mut runner = runner_at(RunStatus::Reviewing);
20875 runner.state.repo = repo.clone();
20876 runner.state.base_commit = base;
20877 runner.state.pr_head = Some("fix/review".into());
20878 runner.state.config.graph.review_rounds = max_rounds;
20879 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20880 runner.state.config.graph.sessions = false;
20881 runner.state.config.disk.min_free_bytes = 0;
20882 runner.state.config.verify.e2e =
20883 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20884 runner.state.candidates = vec![Candidate {
20885 index: 0,
20886 label: 'A',
20887 agent: "fixture".into(),
20888 branch: "fix/review".into(),
20889 worktree: repo.clone(),
20890 summary: String::new(),
20891 stat: String::new(),
20892 files: 1,
20893 commits: 1,
20894 empty: false,
20895 failed: None,
20896 verified_noop: None,
20897 duration_ms: 0,
20898 folded: false,
20899 }];
20900 runner.state.tally = Some(Tally {
20901 first_choice: BTreeMap::new(),
20902 borda: BTreeMap::new(),
20903 winner: 'A',
20904 rankings: 0,
20905 unanimous_initial: true,
20906 deliberated: false,
20907 changed_votes: 0,
20908 unanimous_final: true,
20909 tie_break: None,
20910 judges: 0,
20911 present: 0,
20912 quorum: 0,
20913 met_quorum: true,
20914 uncontested: Some("review-only".into()),
20915 });
20916 let mut last = review_round(false, 0, 1, 1, false, false);
20917 last.head = reviewed.clone();
20918 last.e2e_deferred = true;
20919 last.e2e[0].resource_blocked = true;
20920 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20921 if max_rounds == 1 {
20922 last.e2e.clear();
20923 }
20924 runner.state.reviews = vec![last];
20925 let calls = tmp.path().join("fix-calls");
20926 let mut fixer = spec("repair");
20927 fixer
20928 .env
20929 .insert("FIX_CALLS".into(), calls.display().to_string());
20930 fixer.command = vec!["sh".into(), "-c".into(), r#"
20931 echo fix >> "$FIX_CALLS"
20932 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20933 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20934 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20935 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20936 "#.into()];
20937 let mut reviewer = spec("reviewer");
20938 reviewer.command = vec![
20939 "sh".into(),
20940 "-c".into(),
20941 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20942 .into(),
20943 ];
20944 runner.roles.fixer = Some(vec![fixer]);
20945 runner.roles.reviewers = vec![reviewer];
20946
20947 runner.sync_to_base().await.unwrap();
20948 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20949 assert_ne!(rebased, reviewed);
20950 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20951 if reentry {
20952 let shell = runner.state.config.shell();
20955 runner
20956 .stop_unresolved_review("deferred verification", &shell, &repo)
20957 .await
20958 .unwrap();
20959 assert!(runner.catchup_repairable());
20960 assert!(!calls.exists());
20961 runner.state = RunState::load(&runner.state.id).unwrap();
20962 assert!(runner.catchup_repairable());
20963 }
20964 runner.review_loop().await.unwrap();
20965 let first = &runner.state.reviews[0];
20966 assert_eq!(first.head, reviewed);
20967 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20968 assert!(
20969 first.verify_retried,
20970 "a reproduced build failure gets exactly one retry: {:?}",
20971 first.e2e
20972 );
20973 assert!(first.e2e[0].output_tail.contains("E0063"));
20974 assert!(!runner.catchup_repairable());
20975 if max_rounds == 2 {
20976 assert_eq!(
20977 runner.state.status,
20978 RunStatus::Gating,
20979 "{}",
20980 crate::report::run(&runner.state)
20981 );
20982 assert_eq!(runner.state.reviews.len(), 2);
20983 assert!(first.fix.as_ref().unwrap().committed);
20984 assert!(runner.state.reviews[1].clean);
20985 assert_eq!(
20986 runner
20987 .state
20988 .reviews
20989 .iter()
20990 .filter(|r| r.fix.is_some())
20991 .count(),
20992 1
20993 );
20994 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20995 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20996 } else {
20997 assert_eq!(runner.state.status, RunStatus::Blocked);
20998 assert!(first.fix.is_none());
20999 assert!(!calls.exists());
21000 let recovery = crate::recovery::assess(
21001 "prior",
21002 Some(&runner.state),
21003 &runner.landing_base(),
21004 crate::recovery::Relation::Diverged,
21005 );
21006 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21007 let reasons = recovery.reasons.join("\n");
21008 assert!(
21009 reasons.contains(&reviewed) && reasons.contains(&rebased),
21010 "{reasons}"
21011 );
21012 assert!(
21013 reasons.contains("base sync to")
21014 && reasons.contains("additional review is required"),
21015 "{reasons}"
21016 );
21017 }
21018 }
21019 }
21020
21021 #[tokio::test]
21033 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21034 crate::run::pin_test_home();
21035 let home = crate::run::home();
21036
21037 let tmp = tempfile::tempdir().expect("tempdir");
21038 let repo = tmp.path().join("repo");
21039 std::fs::create_dir_all(&repo).unwrap();
21040 init_repo(&repo);
21041 let head = crate::git::rev_parse(&repo, "HEAD")
21042 .await
21043 .expect("rev-parse");
21044 let cache_dir = tmp.path().join("target");
21045
21046 let mut config = Config::default();
21047 config.verify.e2e = vec![format!(
21048 "CARGO_TARGET_DIR='{}' test -f README.md",
21049 cache_dir.display()
21050 )];
21051 config.graph.review_rounds = 1;
21052 config.graph.timeout_verify = Some(2);
21053 config.graph.timeout_resource_wait = 2;
21054
21055 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21056 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21057 .expect("no io error acquiring directly")
21058 {
21059 crate::cache::AcquireOutcome::Acquired(g) => g,
21060 crate::cache::AcquireOutcome::Busy(b) => {
21061 panic!("expected the direct acquire to win the lease first: {b:?}")
21062 }
21063 };
21064
21065 let mut state = RunState::new(
21066 repo.clone(),
21067 "main".to_owned(),
21068 head.clone(),
21069 "task".to_owned(),
21070 config,
21071 );
21072 state.candidates = vec![Candidate {
21073 index: 0,
21074 label: 'A',
21075 agent: "alpha".to_owned(),
21076 branch: "does-not-exist".to_owned(),
21077 worktree: repo.clone(),
21078 summary: String::new(),
21079 stat: String::new(),
21080 files: 0,
21081 commits: 0,
21082 empty: false,
21083 failed: None,
21084 verified_noop: None,
21085 duration_ms: 0,
21086 folded: false,
21087 }];
21088 state.tally = Some(Tally {
21089 first_choice: BTreeMap::from([('A', 1)]),
21090 borda: BTreeMap::new(),
21091 winner: 'A',
21092 rankings: 1,
21093 unanimous_initial: true,
21094 deliberated: false,
21095 changed_votes: 0,
21096 unanimous_final: true,
21097 tie_break: None,
21098 judges: 0,
21099 present: 0,
21100 quorum: 0,
21101 met_quorum: true,
21102 uncontested: Some("only candidate A produced a change".to_owned()),
21103 });
21104 state.reviews = vec![ReviewRound {
21108 round: 1,
21109 head: head.clone(),
21110 verified_head: Some(head.clone()),
21111 verified_at: Some(jiff::Timestamp::now()),
21112 reviews: Vec::new(),
21113 e2e: vec![CommandOutcome {
21114 command: format!(
21115 "CARGO_TARGET_DIR='{}' test -f README.md",
21116 cache_dir.display()
21117 ),
21118 code: None,
21119 output_tail: "waiting for the shared build cache".to_owned(),
21120 duration_ms: 0,
21121 resource_blocked: true,
21122 }],
21123 fix: None,
21124 blocking: 1,
21125 answered: 1,
21126 expected: 1,
21127 clean: false,
21128 verify_retried: false,
21129 e2e_deferred: false,
21130 e2e_defer_reason: None,
21131 progressed: false,
21132 vote_split: false,
21133 reconsideration: Vec::new(),
21134 verdict: None,
21135 }];
21136
21137 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21138 let mut runner = Runner {
21139 state,
21140 roles: ResolvedRoles {
21141 implementers: Vec::new(),
21142 judges: Vec::new(),
21143 reviewers: Vec::new(),
21144 fixer: None,
21145 conductor: conductor(),
21146 implementer_roster: Vec::new(),
21147 judge_roster: Vec::new(),
21148 reviewer_roster: Vec::new(),
21149 },
21150 sem: Arc::new(Semaphore::new(1)),
21151 pause: Pause::new(),
21152 interrupt: Pause::new(),
21153 };
21154
21155 runner.review_loop().await.expect("review_loop");
21160
21161 assert_eq!(
21162 runner.state.reviews.len(),
21163 1,
21164 "no new round was started on top of the unresolved one: {:?}",
21165 runner.state.reviews
21166 );
21167 let last = &runner.state.reviews[0];
21168 assert_eq!(
21169 last.e2e_status(),
21170 E2eStatus::ResourceBlocked,
21171 "still contended: {last:?}"
21172 );
21173 assert!(
21174 last.verified_at.expect("still known") > first_attempt_at,
21175 "review_loop must have actually retried the check, not left it exactly as found"
21176 );
21177 assert_ne!(
21178 runner.state.status,
21179 RunStatus::Blocked,
21180 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21181 runner.state.status
21182 );
21183
21184 held.release();
21185 }
21186
21187 #[tokio::test]
21188 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21189 crate::run::pin_test_home();
21190 for status in [RunStatus::Blocked, RunStatus::Gating] {
21191 let mut runner = runner_at(status);
21192 runner.state.config.graph.land = true;
21193 runner.state.gate_ran = true;
21194 runner.state.merge = Some(MergeOutcome {
21195 mode: MergeMode::Pr,
21196 ok: false,
21197 detail: "the fixer produced no commit".into(),
21198 empty: false,
21199 });
21200 let url = "https://example.invalid/o/r/pull/1";
21201 runner.state.pr = Some(crate::run::PrRecord {
21202 url: url.into(),
21203 number: 1,
21204 state: "open".into(),
21205 checks: "red".into(),
21206 round: 1,
21207 rounds: 1,
21208 red_at_merge: Vec::new(),
21209 });
21210 runner.state.save().unwrap();
21211 runner.execute_graph().await.unwrap();
21215 assert_eq!(runner.state.status, RunStatus::Blocked);
21216 assert!(runner.state.land_sync_pending);
21217 assert!(runner.state.events.iter().any(|event|
21218 event.node == "land" && event.message == format!("watching {url}")));
21219 assert_eq!(
21220 runner.state.merge.as_ref().unwrap().detail,
21221 "the fixer produced no commit"
21222 );
21223 }
21224 }
21225
21226 #[tokio::test]
21227 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21228 crate::run::pin_test_home();
21229 let tmp = tempfile::tempdir().expect("tempdir");
21230 let repo = tmp.path().join("repo");
21231 std::fs::create_dir_all(&repo).unwrap();
21232 init_repo(&repo);
21233
21234 let mut config = Config::default();
21235 config.merge.mode = MergeMode::Pr;
21236 config.graph.land = true;
21237 config.graph.land_approval = false;
21238
21239 let mut state = RunState::new(
21240 repo.clone(),
21241 "main".to_owned(),
21242 "deadbeef".to_owned(),
21243 "task".to_owned(),
21244 config,
21245 );
21246 state.candidates = vec![Candidate {
21247 index: 0,
21248 label: 'A',
21249 agent: "alpha".to_owned(),
21250 branch: "does-not-exist".to_owned(),
21251 worktree: repo.clone(),
21252 summary: String::new(),
21253 stat: String::new(),
21254 files: 0,
21255 commits: 0,
21256 empty: false,
21257 failed: None,
21258 verified_noop: None,
21259 duration_ms: 0,
21260 folded: false,
21261 }];
21262 state.tally = Some(Tally {
21263 first_choice: BTreeMap::from([('A', 1)]),
21264 borda: BTreeMap::new(),
21265 winner: 'A',
21266 rankings: 1,
21267 unanimous_initial: true,
21268 deliberated: false,
21269 changed_votes: 0,
21270 unanimous_final: true,
21271 tie_break: None,
21272 judges: 0,
21273 present: 0,
21274 quorum: 0,
21275 met_quorum: true,
21276 uncontested: Some("only candidate A produced a change".to_owned()),
21277 });
21278 state.reviews = vec![ReviewRound {
21279 round: 1,
21280 head: "deadbeef".to_owned(),
21281 verified_head: None,
21282 verified_at: None,
21283 reviews: Vec::new(),
21284 e2e: Vec::new(),
21285 fix: None,
21286 blocking: 0,
21287 answered: 0,
21288 expected: 0,
21289 clean: true,
21290 verify_retried: false,
21291 e2e_deferred: false,
21292 e2e_defer_reason: None,
21293 progressed: false,
21294 vote_split: false,
21295 reconsideration: Vec::new(),
21296 verdict: None,
21297 }];
21298 state.gate = vec![CommandOutcome {
21299 command: "test".to_owned(),
21300 code: Some(0),
21301 output_tail: String::new(),
21302 duration_ms: 0,
21303 resource_blocked: false,
21304 }];
21305 state.gate_ran = true;
21306 state.status = RunStatus::Landing;
21310 state.merge = Some(MergeOutcome {
21311 mode: MergeMode::Pr,
21312 ok: true,
21313 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21314 empty: false,
21315 });
21316
21317 ask_test_home();
21321 state.save().expect("save landing run");
21324 let store = ask::Questions::open();
21325 let q = ask_open_question(&store, &state.id);
21326
21327 let mut runner = Runner {
21328 state,
21329 roles: ResolvedRoles {
21330 implementers: Vec::new(),
21331 judges: Vec::new(),
21332 reviewers: Vec::new(),
21333 fixer: None,
21334 conductor: conductor(),
21335 implementer_roster: Vec::new(),
21336 judge_roster: Vec::new(),
21337 reviewer_roster: Vec::new(),
21338 },
21339 sem: Arc::new(Semaphore::new(1)),
21340 pause: Pause::new(),
21341 interrupt: Pause::new(),
21342 };
21343
21344 runner.execute().await.expect("execute");
21349
21350 assert_eq!(
21351 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21352 Some("https://example.invalid/x/y/pull/1"),
21353 "reentry must not push again or open a second pull request over the \
21354 one `land` is already watching"
21355 );
21356 assert_ne!(
21357 runner.state.status,
21358 RunStatus::Landing,
21359 "land could not actually reach the fake pull request, so it must \
21360 have given up rather than left the run silently parked forever"
21361 );
21362 assert_eq!(runner.state.status, RunStatus::Blocked);
21366 assert!(
21367 store.get(&q.id).unwrap().status.open(),
21368 "Blocked is still alive; settle_questions must have been a no-op here"
21369 );
21370 }
21371
21372 fn state_with_round(round: ReviewRound) -> RunState {
21373 let mut s = RunState::new(
21374 PathBuf::from("/repo"),
21375 "main".to_owned(),
21376 "abc1234".to_owned(),
21377 "add retries".to_owned(),
21378 Config::default(),
21379 );
21380 s.reviews = vec![round];
21381 s
21382 }
21383
21384 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21385 crate::verdict::Finding {
21386 id: id.to_owned(),
21387 severity,
21388 file: None,
21389 line: None,
21390 title: title.to_owned(),
21391 detail: String::new(),
21392 }
21393 }
21394
21395 #[test]
21396 fn pr_body_names_open_findings_and_declined_ones() {
21397 let round = ReviewRound {
21398 round: 2,
21399 head: "deadbee".to_owned(),
21400 verified_head: None,
21401 verified_at: None,
21402 reviews: vec![ReviewRecord {
21403 attempts: 0,
21404 reviewer: 1,
21405 agent: "alpha".to_owned(),
21406 summary: String::new(),
21407 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21408 vote: None,
21409 failed: None,
21410 duration_ms: 0,
21411 }],
21412 e2e: vec![CommandOutcome {
21413 command: "cargo test".to_owned(),
21414 code: Some(0),
21415 output_tail: String::new(),
21416 duration_ms: 0,
21417 resource_blocked: false,
21418 }],
21419 verify_retried: false,
21420 e2e_deferred: false,
21421 e2e_defer_reason: None,
21422 fix: Some(FixRecord {
21423 agent: "alpha".to_owned(),
21424 addressed: Vec::new(),
21425 rejected: vec![crate::verdict::Rejection {
21426 id: "R1-1-1".to_owned(),
21427 why: "not reachable from any caller".to_owned(),
21428 }],
21429 notes: String::new(),
21430 committed: true,
21431 failed: None,
21432 duration_ms: 0,
21433 continuation: None,
21434 }),
21435 blocking: 0,
21436 answered: 1,
21437 expected: 1,
21438 clean: false,
21439 progressed: true,
21440 vote_split: false,
21441 reconsideration: Vec::new(),
21442 verdict: None,
21443 };
21444 let state = state_with_round(round);
21445 let body = pr_message(&state, 'A').body;
21446
21447 assert!(body.contains("add retries"), "the task must still be there");
21448 assert!(body.contains("R2-1-1"), "{body}");
21449 assert!(!body.contains("unused import"), "{body}");
21450 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21451 assert!(
21452 !body.contains("not reachable from any caller"),
21453 "the reason it was declined: {body}"
21454 );
21455 }
21456
21457 #[test]
21458 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21459 let round = ReviewRound {
21460 round: 1,
21461 head: "deadbee".to_owned(),
21462 verified_head: None,
21463 verified_at: None,
21464 reviews: vec![ReviewRecord {
21465 attempts: 0,
21466 reviewer: 1,
21467 agent: "alpha".to_owned(),
21468 summary: String::new(),
21469 findings: Vec::new(),
21470 vote: None,
21471 failed: None,
21472 duration_ms: 0,
21473 }],
21474 e2e: Vec::new(),
21475 verify_retried: false,
21476 e2e_deferred: false,
21477 e2e_defer_reason: None,
21478 fix: None,
21479 blocking: 0,
21480 answered: 1,
21481 expected: 1,
21482 clean: true,
21483 progressed: false,
21484 vote_split: false,
21485 reconsideration: Vec::new(),
21486 verdict: None,
21487 };
21488 let state = state_with_round(round);
21489 let body = pr_message(&state, 'A').body;
21490 assert!(!body.contains("Open review findings"), "{body}");
21491 assert!(!body.contains("Declined"), "{body}");
21492 }
21493
21494 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21495 let mut state = RunState::new(
21496 PathBuf::from("/repo"),
21497 "main".to_owned(),
21498 "abc1234".to_owned(),
21499 instruction.to_owned(),
21500 Config::default(),
21501 );
21502 state.config.disk.min_free_bytes = 0;
21504 state.candidates.push(Candidate {
21505 index: 0,
21506 label: 'A',
21507 agent: "alpha".to_owned(),
21508 branch: "magi/x/A".to_owned(),
21509 worktree: PathBuf::from("/wt"),
21510 summary: summary.to_owned(),
21511 stat: String::new(),
21512 files: 1,
21513 commits: 1,
21514 empty: false,
21515 failed: None,
21516 verified_noop: None,
21517 folded: false,
21518 duration_ms: 0,
21519 });
21520 state
21521 }
21522
21523 fn review_state(subjects: &[&str]) -> RunState {
21524 let mut state = state_with_summary(
21525 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21526 "",
21527 );
21528 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21529 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21530 state
21531 }
21532
21533 #[tokio::test]
21536 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21537 ask_test_home();
21538 let tmp = tempfile::tempdir().unwrap();
21539 let repo = tmp.path().join("repo");
21540 std::fs::create_dir_all(&repo).unwrap();
21541 init_repo(&repo);
21542 let origin = tmp.path().join("origin.git");
21543 let g = |dir: &Path, args: &[&str]| {
21544 let out = std::process::Command::new("git")
21545 .args(args)
21546 .current_dir(dir)
21547 .quiet()
21548 .output()
21549 .expect("spawn git");
21550 assert!(
21551 out.status.success(),
21552 "git {args:?}: {}",
21553 String::from_utf8_lossy(&out.stderr)
21554 );
21555 };
21556 g(
21557 tmp.path(),
21558 &[
21559 "clone",
21560 "--bare",
21561 "-q",
21562 repo.to_str().unwrap(),
21563 origin.to_str().unwrap(),
21564 ],
21565 );
21566 g(
21567 &repo,
21568 &["remote", "add", "origin", origin.to_str().unwrap()],
21569 );
21570 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21571
21572 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21574 g(&repo, &["add", "-A"]);
21575 g(
21576 &repo,
21577 &["commit", "-q", "-m", "chore(deps): update a crate"],
21578 );
21579 g(&repo, &["push", "-q", "origin", "main"]);
21580 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21581 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21582 g(&repo, &["add", "-A"]);
21583 g(
21584 &repo,
21585 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21586 );
21587 g(&repo, &["checkout", "-q", "main"]);
21588
21589 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21590 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21591 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21593 assert_eq!(fallback, c1);
21594
21595 let mut state = review_state(&[
21596 "chore(deps): update a crate",
21597 "fix(daemon): apply a chosen action",
21598 ]);
21599 state.repo = repo.clone();
21600 state.base_branch = "main".to_owned();
21601 state.base_commit = c1;
21602 refresh_reviewed_commits(&mut state, "feat/own").await;
21603 assert_eq!(
21604 state.reviewed_commits,
21605 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21606 );
21607 assert_eq!(
21608 review_title(&state).as_deref(),
21609 Some("fix(daemon): apply a chosen action")
21610 );
21611 assert_eq!(
21612 leaked_subjects(&state, "feat/own").await,
21613 Some(vec!["chore(deps): update a crate".to_owned()])
21614 );
21615 state.config.merge.remote = "nowhere".to_owned();
21618 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21619 }
21620
21621 struct SummaryFixture {
21625 _tmp: tempfile::TempDir,
21626 repo: PathBuf,
21627 reply: PathBuf,
21628 runner: Runner,
21629 }
21630
21631 fn summary_fixture() -> SummaryFixture {
21632 ask_test_home();
21633 let tmp = tempfile::tempdir().unwrap();
21634 let repo = tmp.path().join("repo");
21635 std::fs::create_dir_all(&repo).unwrap();
21636 init_repo(&repo);
21637 let origin = tmp.path().join("origin.git");
21638 let g = |args: &[&str]| {
21639 let out = std::process::Command::new("git")
21640 .args(args)
21641 .current_dir(&repo)
21642 .quiet()
21643 .output()
21644 .expect("spawn git");
21645 assert!(
21646 out.status.success(),
21647 "git {args:?}: {}",
21648 String::from_utf8_lossy(&out.stderr)
21649 );
21650 };
21651 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21652 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21653 g(&["push", "-q", "origin", "main"]);
21654 g(&["checkout", "-q", "-b", "feat/own"]);
21655 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21656 g(&["add", "-A"]);
21657 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21658 let reply = tmp.path().join("reply.txt");
21659 std::fs::write(
21660 &reply,
21661 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21662 )
21663 .unwrap();
21664 let mut runner = runner_at(RunStatus::Gating);
21665 let mut state = review_state(&["feat(own): add the own file"]);
21666 state.repo = repo.clone();
21667 state.base_branch = "main".to_owned();
21668 state.candidates[0].branch = "feat/own".to_owned();
21669 state.candidates[0].worktree = repo.clone();
21670 let mut author = spec("alpha");
21671 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21672 author
21673 .env
21674 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21675 state.config.agents = vec![author];
21676 runner.state = state;
21677 SummaryFixture {
21678 _tmp: tmp,
21679 repo,
21680 reply,
21681 runner,
21682 }
21683 }
21684
21685 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21686 let winner = f.runner.state.candidates[0].clone();
21687 f.runner
21688 .guarded_pr_message(&winner, None, true)
21689 .await
21690 .unwrap()
21691 .expect("an English description never parks")
21692 }
21693
21694 fn clean_round_with_minor_findings() -> ReviewRound {
21695 let mut round = passing_round(2, None);
21696 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21697 round.reviews = vec![ReviewRecord {
21698 attempts: 0,
21699 reviewer: 1,
21700 agent: "alpha".into(),
21701 summary: "内部レビュー".into(),
21702 findings: vec![
21703 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21704 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21705 ],
21706 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21707 failed: None,
21708 duration_ms: 0,
21709 }];
21710 round
21711 }
21712
21713 #[tokio::test]
21714 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21715 for review_only in [false, true] {
21716 let mut f = summary_fixture();
21717 if !review_only {
21718 f.runner.state.candidates[0].agent = "alpha".into();
21719 }
21720 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21721 assert!(f.runner.state.reviews[0].clean);
21722 assert!(f.runner.state.reviews[0].fix.is_none());
21723 assert!(f.runner.state.open_findings().is_empty());
21724 assert_eq!(
21725 crate::pr_outcome::required_finding_ids(&f.runner.state),
21726 ["R2-1-1", "R2-1-2"]
21727 );
21728 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21729 let evidence = crate::pr_outcome::summary_evidence(
21730 &f.runner.state,
21731 crate::pr_outcome::FinalHead::At(&head),
21732 );
21733 assert!(
21734 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21735 );
21736 let omitted = posted(&mut f).await;
21738 assert!(omitted.body.contains("No description"));
21739 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21740 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21741 assert!(f.runner.state.change_summary.is_none());
21742 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21743 let described = posted(&mut f).await;
21744 assert!(described.body.contains("improve documentation for callers"));
21745 assert!(described.body.contains("remaining edge case"));
21746 assert!(described.body.contains("## Open review findings"));
21747 assert!(!described.body.contains("未修正"));
21748 }
21749 }
21750
21751 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21752 let mut f = summary_fixture();
21753 if !review_only {
21754 f.runner.state.candidates[0].agent = "alpha".into();
21755 }
21756 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21757 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21758 fixed
21759 .fix
21760 .as_mut()
21761 .unwrap()
21762 .rejected
21763 .push(crate::verdict::Rejection {
21764 id: "R1-1-1".into(),
21765 why: "呼び出されないコードのため辞退".into(),
21766 });
21767 let mut clean = clean_round_with_minor_findings();
21768 clean.verified_head = Some(head.clone());
21769 f.runner.state.reviews = vec![fixed, clean];
21770 ok_gate(&mut f.runner.state, Some(&head));
21771 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21772 f.runner.state.config.agents.push(scripted("beta", reply));
21773 f.runner.state.config.roles.language_judge =
21774 Some(crate::config::AgentChoice::One("beta".into()));
21775 f
21776 }
21777
21778 #[tokio::test]
21779 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21780 for review_only in [false, true] {
21781 let mut f = rewrite_fixture_with_findings(review_only,
21782 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21783 let m = posted(&mut f).await;
21784 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21785 let expected =
21786 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21787 assert!(m.body.contains(&expected));
21788 assert!(m.body.contains("## Review rounds"));
21789 assert!(
21790 m.body
21791 .contains("Gate (`verify.gate`): passed on the final commit")
21792 );
21793 assert!(
21794 m.body
21795 .contains("Tests (`verify.e2e`): passed on the final commit")
21796 );
21797 let c = f.runner.state.change_summary.as_ref().unwrap();
21798 assert!(
21799 m.body
21800 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21801 );
21802 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21803 assert!(crate::pr_outcome::covers_required_findings(
21804 &f.runner.state,
21805 &m.body
21806 ));
21807 assert!(m.body.contains("documentation remains incomplete"));
21808 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21809 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21810 assert!(f.runner.state.github_text.is_none());
21811 }
21812 }
21813
21814 #[tokio::test]
21815 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21816 for review_only in [false, true] {
21817 for reply in [
21818 "TITLE: fix: public description\nA short English description.",
21819 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21820 ] {
21821 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21822 let winner = f.runner.state.candidates[0].clone();
21823 let posted = f
21824 .runner
21825 .guarded_pr_message(&winner, None, true)
21826 .await
21827 .unwrap();
21828 assert!(posted.is_none());
21829 assert!(f.runner.state.parked);
21830 let withheld = f.runner.state.github_text.as_ref().unwrap();
21831 assert!(
21832 withheld.rewrite.is_none(),
21833 "an incomplete rewrite must not be selectable"
21834 );
21835 let q = ask::Questions::open()
21836 .get(withheld.question.as_ref().unwrap())
21837 .unwrap();
21838 assert!(
21839 !q.choices
21840 .contains(&crate::github_text::USE_REWRITE.to_owned())
21841 );
21842 assert!(
21843 q.detail.contains("R1-1-1")
21844 && q.detail.contains("R2-1-1")
21845 && q.detail.contains("R2-1-2")
21846 );
21847 }
21848 }
21849 }
21850
21851 #[tokio::test]
21852 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21853 for review_only in [false, true] {
21854 let mut f = summary_fixture();
21855 if !review_only {
21856 f.runner.state.candidates[0].agent = "alpha".into();
21857 }
21858 f.runner.state.instruction = "内部タスクの説明".into();
21859 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21860 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21861 f.runner.state.reviews = vec![
21862 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21863 round_with_notes(2, Some("二回目の内部修正報告です。")),
21864 passing_round(3, Some(&head)),
21865 ];
21866 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21867 ok_gate(&mut f.runner.state, Some(&head));
21868 let captured = f.repo.join("public-prompt.txt");
21869 let author = &mut f.runner.state.config.agents[0];
21870 author.command = vec![
21871 "sh".into(),
21872 "-c".into(),
21873 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21874 ];
21875 author.env.insert(
21876 "PROMPT_FILE".into(),
21877 captured.to_string_lossy().into_owned(),
21878 );
21879 let m = posted(&mut f).await;
21880 assert!(m.body.contains("Adds `own.txt`"));
21881 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21882 assert!(m.body.contains("R1-1-1"));
21883 assert!(
21884 m.body
21885 .contains("Gate (`verify.gate`): passed on the final commit")
21886 );
21887 assert!(
21888 m.body
21889 .contains("Tests (`verify.e2e`): passed on the final commit")
21890 );
21891 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21892 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21893 let prompt = std::fs::read_to_string(captured).unwrap();
21894 assert!(prompt.contains(&format!(
21895 "git diff {} {head}",
21896 f.runner.state.change_summary.as_ref().unwrap().base
21897 )));
21898 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21899 assert_eq!(
21900 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21901 "二回目の内部修正報告です。"
21902 );
21903 }
21904 }
21905
21906 #[tokio::test]
21907 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21908 for review_only in [false, true] {
21909 let mut f = summary_fixture();
21910 if !review_only {
21911 f.runner.state.candidates[0].agent = "alpha".into();
21912 }
21913 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21914 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21915 std::fs::write(&f.reply, "no usable description").unwrap();
21916 let failed = posted(&mut f).await;
21917 assert!(
21918 failed
21919 .body
21920 .contains("No description of what this change does could be generated")
21921 );
21922 assert!(failed.body.is_ascii());
21923 assert!(f.runner.state.github_text.is_none());
21924 assert!(f.runner.state.change_summary.is_none());
21925 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21926 let evidence = crate::pr_outcome::summary_evidence(
21927 &f.runner.state,
21928 crate::pr_outcome::FinalHead::At(&head),
21929 );
21930 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21931 assert!(
21932 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21933 .exists()
21934 );
21935 f.runner.state =
21936 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21937 std::fs::write(
21938 &f.reply,
21939 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21940 )
21941 .unwrap();
21942 let retried = posted(&mut f).await;
21943 assert!(retried.body.contains("Adds the committed own file."));
21944 assert_eq!(
21945 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21946 "日本語の修正報告"
21947 );
21948 }
21949 }
21950
21951 #[tokio::test]
21952 async fn same_head_verification_and_findings_invalidate_public_prose() {
21953 let mut f = summary_fixture();
21954 posted(&mut f).await;
21955 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21956 let original = f
21957 .runner
21958 .state
21959 .change_summary
21960 .as_ref()
21961 .unwrap()
21962 .evidence
21963 .clone();
21964 ok_gate(&mut f.runner.state, Some(&head));
21965 std::fs::write(
21966 &f.reply,
21967 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21968 )
21969 .unwrap();
21970 let verified = posted(&mut f).await;
21971 assert!(verified.body.contains("now verified"));
21972 assert_ne!(
21973 f.runner.state.change_summary.as_ref().unwrap().evidence,
21974 original
21975 );
21976 let mut round = round_with_notes(1, Some("内部報告"));
21977 round.reviews.push(ReviewRecord {
21978 attempts: 0,
21979 reviewer: 1,
21980 agent: "alpha".into(),
21981 summary: "内部レビュー".into(),
21982 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21983 vote: None,
21984 failed: None,
21985 duration_ms: 0,
21986 });
21987 round
21988 .fix
21989 .as_mut()
21990 .unwrap()
21991 .rejected
21992 .push(crate::verdict::Rejection {
21993 id: "R1-1-1".into(),
21994 why: "呼び出し元から到達できない".into(),
21995 });
21996 f.runner.state.reviews = vec![round];
21997 let omitted = posted(&mut f).await;
21999 assert!(omitted.body.contains("No description"));
22000 assert!(omitted.body.contains("R1-1-1"));
22001 assert!(!omitted.body.contains("到達"));
22002 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22003 let described = posted(&mut f).await;
22004 assert!(described.body.contains("unreachable from callers"));
22005 assert!(described.body.contains("R1-1-1"));
22006 assert!(!described.body.contains("到達"));
22007 let evidence = f
22008 .runner
22009 .state
22010 .change_summary
22011 .as_ref()
22012 .unwrap()
22013 .evidence
22014 .clone();
22015 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22016 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22017 assert!(posted(&mut f).await.body.contains("rationale updated"));
22018 assert_ne!(
22019 f.runner.state.change_summary.as_ref().unwrap().evidence,
22020 evidence
22021 );
22022 f.runner
22024 .state
22025 .change_summary
22026 .as_mut()
22027 .unwrap()
22028 .evidence
22029 .clear();
22030 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22031 assert!(posted(&mut f).await.body.contains("current specification"));
22032 }
22033
22034 #[tokio::test]
22035 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22036 let mut f = summary_fixture();
22037 let m = posted(&mut f).await;
22038 assert_eq!(m.title, "feat(own): add the own file");
22039 assert!(
22040 m.body
22041 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22042 "{}",
22043 m.body
22044 );
22045 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22046 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22047 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22048 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22049 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22050 let kept = f.runner.state.change_summary.as_ref().unwrap();
22051 assert_eq!(
22052 (kept.head.as_str(), kept.base.as_str()),
22053 (head.as_str(), base.as_str())
22054 );
22055 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22056 }
22057
22058 #[tokio::test]
22059 async fn review_only_summary_receives_the_configured_github_language() {
22060 let mut f = summary_fixture();
22061 f.runner.state.config.graph.github_language = "ja".into();
22062 let captured = f.repo.join("summary-prompt.txt");
22063 let author = &mut f.runner.state.config.agents[0];
22064 author.command = vec![
22065 "sh".into(),
22066 "-c".into(),
22067 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22068 ];
22069 author.env.insert(
22070 "PROMPT_FILE".into(),
22071 captured.to_string_lossy().into_owned(),
22072 );
22073 std::fs::write(
22074 &f.reply,
22075 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22076 )
22077 .unwrap();
22078 let m = posted(&mut f).await;
22079 let prompt = std::fs::read_to_string(captured).unwrap();
22080 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22081 assert!(!prompt.contains("in English"));
22082 assert_eq!(m.title, "feat(own): ファイルを追加");
22083 assert!(m.body.contains("own.txt を追加します。"));
22084 }
22085
22086 #[tokio::test]
22087 async fn review_fixes_come_after_the_whole_change_summary() {
22088 let mut f = summary_fixture();
22089 f.runner.state.reviews = vec![
22090 round_with_notes(1, Some("handled the PATH case")),
22091 round_with_notes(2, None),
22092 ];
22093 let m = posted(&mut f).await;
22094 let summary = m.body.find("## Summary").expect("summary");
22095 let fixes = m.body.find("## Review rounds").expect("review rounds");
22096 assert!(summary < fixes, "{}", m.body);
22097 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22098 assert!(m.body.contains("## Final verification"), "{}", m.body);
22099 }
22100
22101 #[tokio::test]
22102 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22103 let mut f = summary_fixture();
22104 let first = posted(&mut f).await;
22105 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22106 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22108 let again = posted(&mut f).await;
22109 assert_eq!((again.title, again.body), (first.title, first.body));
22110
22111 let g = |args: &[&str]| {
22112 let out = std::process::Command::new("git")
22113 .args(args)
22114 .current_dir(&f.repo)
22115 .quiet()
22116 .output()
22117 .expect("spawn git");
22118 assert!(out.status.success(), "git {args:?}");
22119 };
22120 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22121 g(&["add", "-A"]);
22122 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22123 let second = posted(&mut f).await;
22124 let kept = f.runner.state.change_summary.clone().unwrap();
22125 assert_ne!(kept.head, first_head);
22126 assert!(second.body.contains("Not read yet."), "{}", second.body);
22127 assert!(!second.body.contains("so the first summary is read"));
22128 assert_eq!(second.title, "feat(own): other");
22129 }
22130
22131 #[tokio::test]
22132 async fn an_unavailable_summary_is_stated_and_never_invented() {
22133 let mut f = summary_fixture();
22134 std::fs::write(&f.reply, "no title line here").unwrap();
22135 let m = posted(&mut f).await;
22136 assert!(
22137 m.body
22138 .contains("No description of what this change does could be generated"),
22139 "{}",
22140 m.body
22141 );
22142 assert!(m.body.contains("internal reports remain in local records"));
22143 assert!(f.runner.state.change_summary.is_none());
22144 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22145 let winner = f.runner.state.candidates[0].clone();
22147 let preview = f
22148 .runner
22149 .guarded_pr_message(&winner, None, false)
22150 .await
22151 .unwrap()
22152 .unwrap();
22153 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22154 }
22155
22156 #[test]
22157 fn pr_message_review_single_commit_uses_its_subject() {
22158 let state = review_state(&["feat(nats): per-role user"]);
22159 let m = pr_message(&state, 'A');
22160 assert_eq!(m.title, "feat(nats): per-role user");
22161 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22162 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22163 }
22164
22165 #[test]
22166 fn pr_message_review_multi_commit_takes_the_oldest() {
22167 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22168 let m = pr_message(&state, 'A');
22169 assert_eq!(m.title, "feat: the change");
22170 for s in ["feat: the change", "fix: typo", "fix: again"] {
22171 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22172 }
22173 }
22174
22175 #[test]
22176 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22177 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22178 let mut state = review_state(&[first, "fix: later fixup"]);
22179 state.candidates[0].branch = "機能/ログイン".to_owned();
22180 let m = pr_message(&state, 'A');
22181 assert!(
22182 m.title.starts_with("chore: land candidate A of run"),
22183 "{}",
22184 m.title
22185 );
22186 }
22187 }
22188
22189 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22190 BranchFacts {
22191 commits: commits
22192 .iter()
22193 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22194 .collect(),
22195 stat: stat.to_owned(),
22196 }
22197 }
22198
22199 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22200 let mut r = review_round(true, 0, 1, 1, true, true);
22201 r.round = round;
22202 r.fix = notes.map(|n| FixRecord {
22203 agent: "fixer".to_owned(),
22204 addressed: Vec::new(),
22205 rejected: Vec::new(),
22206 notes: n.to_owned(),
22207 committed: true,
22208 failed: None,
22209 duration_ms: 0,
22210 continuation: None,
22211 });
22212 r
22213 }
22214
22215 #[test]
22216 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22217 let state = review_state(&["ignored"]);
22218 let f = facts(
22219 &[
22220 (
22221 "fix(login): resolve PATH on macOS",
22222 "Login shells skip rc files.",
22223 ),
22224 ("fix: address review", ""),
22225 ],
22226 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22227 );
22228 let m = pr_message_with(&state, 'A', Some(&f));
22229 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22230 assert!(
22231 m.body
22232 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22233 "{}",
22234 m.body
22235 );
22236 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22237 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22238 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22239 for banned in [
22240 "Review the work already",
22241 "no task statement",
22242 "Original task",
22243 ] {
22244 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22245 }
22246 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22247 }
22248
22249 #[test]
22250 fn pr_message_review_truncates_a_huge_first_commit_body() {
22251 let state = review_state(&["ignored"]);
22252 let f = facts(
22253 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22254 "s",
22255 );
22256 let m = pr_message_with(&state, 'A', Some(&f));
22257 assert!(m.body.len() < 30_000, "{}", m.body.len());
22258 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22259 assert!(
22260 m.body.contains("- ... 1 more commit(s)"),
22261 "{}",
22262 m.body.len()
22263 );
22264 assert!(m.body.ends_with("magi:candidate-a\n"));
22265 }
22266
22267 #[test]
22268 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22269 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22270 assert_eq!(m.title, "feat: x");
22271 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22272 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22273 assert!(!m.body.contains("no task statement"), "{}", m.body);
22274 }
22275
22276 #[test]
22277 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22278 let mut state = review_state(&["日本語の件名"]);
22279 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22280 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22281 state.candidates[0].branch = "機能/ログイン".to_owned();
22282 assert!(
22283 pr_message(&state, 'A')
22284 .title
22285 .starts_with("chore: land candidate A")
22286 );
22287 }
22288
22289 #[test]
22290 fn pr_message_review_fixes_survive_a_clean_final_round() {
22291 let mut state = review_state(&["feat: x"]);
22292 state.reviews = vec![
22293 round_with_notes(1, Some("handled the PATH case")),
22294 round_with_notes(2, None),
22295 ];
22296 let body = pr_message(&state, 'A').body;
22297 assert!(
22298 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22299 "{body}"
22300 );
22301 assert!(body.contains("### Round 1"), "{body}");
22302
22303 state.reviews = vec![
22304 round_with_notes(1, Some("first fix")),
22305 round_with_notes(2, Some("")),
22306 round_with_notes(3, Some("second fix")),
22307 round_with_notes(4, None),
22308 ];
22309 let body = pr_message(&state, 'A').body;
22310 assert!(
22311 body.contains("### Round 1") && !body.contains("first fix"),
22312 "{body}"
22313 );
22314 assert!(
22315 body.contains("### Round 3") && !body.contains("second fix"),
22316 "{body}"
22317 );
22318 assert!(body.contains("### Round 2"), "{body}");
22319 }
22320
22321 const FINAL: &str = "abcdef1234567890";
22322
22323 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22324 let mut r = round_with_notes(round, None);
22325 r.verified_head = verified.map(str::to_owned);
22326 r
22327 }
22328
22329 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22330 state.gate = vec![CommandOutcome {
22331 command: "gate".to_owned(),
22332 code: Some(0),
22333 output_tail: String::new(),
22334 duration_ms: 0,
22335 resource_blocked: false,
22336 }];
22337 state.gate_ran = true;
22338 state.gate_head = head.map(str::to_owned);
22339 }
22340
22341 fn body_at(state: &RunState) -> String {
22342 pr_message_raw(
22343 state,
22344 'A',
22345 None,
22346 crate::pr_outcome::FinalHead::At(FINAL),
22347 ChangeNote::NotAsked,
22348 )
22349 .body
22350 }
22351
22352 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22353
22354 #[test]
22355 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22356 let mut state = review_state(&["feat: x"]);
22357 state.reviews = vec![
22358 round_with_notes(1, Some(SANDBOXED)),
22359 passing_round(2, Some(FINAL)),
22360 ];
22361 ok_gate(&mut state, Some(FINAL));
22362 let body = body_at(&state);
22363 assert!(!body.contains("Handled the PATH case."), "{body}");
22364 assert!(!body.contains("could not commit"), "{body}");
22365 assert!(!body.contains("could not run"), "{body}");
22366 assert!(
22367 body.contains("The fix is committed on the branch"),
22368 "{body}"
22369 );
22370 assert!(!body.contains("sandbox"), "{body}");
22371 assert!(
22372 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22373 "{body}"
22374 );
22375 assert!(
22376 body.contains("Gate (`verify.gate`): passed on the final commit"),
22377 "{body}"
22378 );
22379 assert!(
22381 state.reviews[0]
22382 .fix
22383 .as_ref()
22384 .unwrap()
22385 .notes
22386 .contains("could not commit")
22387 );
22388 }
22389
22390 #[test]
22391 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22392 let mut state = review_state(&["feat: x"]);
22393 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22394 r1.fix.as_mut().unwrap().committed = false;
22395 let mut r2 = passing_round(2, Some(FINAL));
22396 r2.e2e[0].code = Some(1);
22397 state.reviews = vec![r1, r2];
22398 let body = body_at(&state);
22399 assert!(!body.contains("could not commit"), "{body}");
22400 assert!(!body.contains("could not run"), "{body}");
22401 assert!(body.contains("This round produced no commit"), "{body}");
22402 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22403 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22404 assert!(!body.contains("passed"), "{body}");
22405 }
22406
22407 #[test]
22408 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22409 let mut state = review_state(&["feat: x"]);
22410 state.reviews = vec![
22411 round_with_notes(1, Some(SANDBOXED)),
22412 passing_round(2, Some("0123456789abcdef")),
22413 ];
22414 ok_gate(&mut state, Some("0123456789abcdef"));
22415 let body = body_at(&state);
22416 assert!(!body.contains("HTTP tests could not run"), "{body}");
22417 assert!(!body.contains("could not commit"), "{body}");
22419 assert!(
22420 body.contains("passed on an earlier commit (`0123456`)"),
22421 "{body}"
22422 );
22423 assert!(!body.contains("passed on the final commit"), "{body}");
22424
22425 ok_gate(&mut state, None);
22427 let body = body_at(&state);
22428 assert!(
22429 body.contains("passed on a commit that was not recorded"),
22430 "{body}"
22431 );
22432 }
22433
22434 #[test]
22435 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22436 let state = state_with_summary(
22437 "add retries to the client",
22438 "TITLE: feat: retries\n\nDid it.",
22439 );
22440 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22441 assert_eq!(m.title, "feat: retries");
22442 assert!(
22443 m.body.contains("<summary>Original task</summary>"),
22444 "{}",
22445 m.body
22446 );
22447 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22448 assert!(
22449 m.body
22450 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22451 "{}",
22452 m.body
22453 );
22454 }
22455
22456 #[test]
22457 fn pr_message_review_bounds_a_long_english_subject() {
22458 let long = format!("feat: {}", "word ".repeat(100));
22459 let m = pr_message(&review_state(&[&long]), 'A');
22460 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22461 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22462 }
22463
22464 #[test]
22465 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22466 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22467 let m = pr_message(&state, 'A');
22468 assert_eq!(m.title, "add retries");
22469 assert!(m.body.contains("<summary>Original task</summary>"));
22470 assert!(!m.body.contains("Commits under review"));
22471 assert_eq!(landing_subject_source(&state), state.instruction);
22472 }
22473
22474 #[test]
22475 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22476 let state = review_state(&["feat: the change", "fix: typo"]);
22477 let source = landing_subject_source(&state);
22478 assert_eq!(land::merge_subject("", &source), "feat: the change");
22479 assert_eq!(
22480 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22481 "feat: the change"
22482 );
22483 assert_eq!(
22485 land::merge_subject("feat: renamed by hand", &source),
22486 "feat: renamed by hand"
22487 );
22488 let blank = review_state(&["日本語"]);
22489 assert!(
22490 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22491 );
22492 }
22493
22494 #[test]
22495 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22496 let state = review_state(&["feat: the change"]);
22497 let source = landing_subject_source(&state);
22498 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22499 assert_eq!(
22500 land::merge_subject(landing_title(&state, old), &source),
22501 "feat: the change"
22502 );
22503 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22504 let task = state_with_summary("add retries", "");
22505 assert_eq!(landing_title(&task, old), old);
22506 }
22507
22508 #[test]
22509 fn pr_message_describes_the_change_not_the_task() {
22510 let state = state_with_summary(
22511 "今回やってほしいこと: results projector を直す",
22512 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22513 );
22514 let m = pr_message(&state, 'A');
22515 assert_eq!(m.title, "fix(web): batch the runs list reads");
22516 assert!(
22517 m.body.starts_with("## Summary\n\n- reads run.json once"),
22518 "{}",
22519 m.body
22520 );
22521 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22522 let task_at = m.body.find("今回やってほしいこと").unwrap();
22523 let details_at = m.body.find("<details>").unwrap();
22524 assert!(
22525 details_at < task_at,
22526 "the task lives inside <details>: {}",
22527 m.body
22528 );
22529 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22530 assert!(m.body.contains("magi:candidate-a"));
22531 }
22532
22533 #[test]
22534 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22535 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22536 let m = pr_message(&state, 'A');
22537 assert_eq!(m.title, "add retries");
22538 assert!(
22539 m.body.contains("## Summary\n\n- did some things"),
22540 "{}",
22541 m.body
22542 );
22543
22544 let none = RunState::new(
22545 PathBuf::from("/repo"),
22546 "main".to_owned(),
22547 "abc1234".to_owned(),
22548 "add retries".to_owned(),
22549 Config::default(),
22550 );
22551 let m = pr_message(&none, 'A');
22552 assert_eq!(m.title, "add retries");
22553 assert!(!m.body.contains("## Summary"), "{}", m.body);
22554 }
22555
22556 #[test]
22557 fn pr_message_refuses_the_candidate_commit_subject() {
22558 for bad in [
22559 "TITLE: magi: candidate A (uncommitted work)",
22560 "TITLE: chore: stuff (uncommitted work)",
22561 "TITLE: ",
22562 ] {
22563 let state = state_with_summary("add retries", bad);
22564 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22565 }
22566 }
22567
22568 #[test]
22569 fn pr_message_bounds_a_very_long_task_and_title() {
22570 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22571 let state = state_with_summary(&long, "- nothing");
22572 let m = pr_message(&state, 'A');
22573 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22574 assert!(!m.title.contains('\n'));
22575
22576 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22577 let m = pr_message(&state, 'A');
22578 assert!(m.title.starts_with("feat: "));
22579 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22580 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22581 }
22582
22583 fn long_title_of(instruction: &str) -> String {
22584 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22585 }
22586
22587 #[test]
22588 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22589 let first = "Make the landing path keep a readable title for long tasks";
22590 let line = format!(
22591 "{first}. {}",
22592 "And then keep going with more words ".repeat(20)
22593 );
22594 let t = long_title_of(&line);
22595 assert_eq!(t, first);
22596 assert!(!t.starts_with("chore: land"));
22597 }
22598
22599 #[test]
22600 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22601 let line = "word ".repeat(200);
22602 let t = long_title_of(&line);
22603 assert!(t.ends_with("word..."), "{t}");
22604 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22605 }
22606
22607 #[test]
22608 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22609 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22610 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22611 }
22612 }
22613
22614 #[test]
22615 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22616 let mut state = state_with_summary("再試行を追加する", "- nothing");
22617 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22618 state.config.graph.github_language = "ja".into();
22619 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22620 }
22621
22622 #[test]
22623 fn pr_message_title_limit_is_exact() {
22624 let at = "a".repeat(PR_TITLE_MAX);
22625 assert_eq!(long_title_of(&at), at);
22626 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22627 assert!(over.ends_with("..."), "{over}");
22628 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22629 }
22630
22631 #[test]
22632 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22633 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22634 let t = long_title_of(&line);
22635 assert!(t.ends_with("..."), "{t}");
22636 assert!(t.is_ascii(), "{t}");
22637 }
22638
22639 #[test]
22640 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22641 let line = format!(
22642 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22643 "plus more filler words ".repeat(20)
22644 );
22645 let t = long_title_of(&line);
22646 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22647 }
22648
22649 #[test]
22650 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22651 let line = format!("\n\n# {}", "title words ".repeat(40));
22652 let state = state_with_summary(&line, "- nothing");
22653 let m = pr_message(&state, 'A');
22654 assert!(m.title.starts_with("title words"), "{}", m.title);
22655 assert_eq!(
22656 land::merge_subject(&m.title, &landing_subject_source(&state)),
22657 m.title
22658 );
22659 assert_eq!(
22661 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22662 "feat: renamed by hand"
22663 );
22664 }
22665
22666 #[tokio::test]
22667 async fn github_text_rewrite_is_bounded_and_falls_back() {
22668 crate::run::pin_test_home();
22669 for (reply, accepted) in [
22670 (
22671 "TITLE: fix: retries\nAdd retries for failed requests.",
22672 true,
22673 ),
22674 (
22675 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22676 false,
22677 ),
22678 (
22679 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22680 true,
22681 ),
22682 (
22683 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22684 true,
22685 ),
22686 (
22687 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22688 true,
22689 ),
22690 (
22691 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22692 false,
22693 ),
22694 ("TITLE: fix: retries\n", false),
22695 ] {
22696 let dir = tempfile::tempdir().unwrap();
22697 let mut runner = runner_at(RunStatus::Gating);
22698 runner.state = state_with_summary(
22699 "add retries",
22700 "TITLE: fix: retries\n日本語の説明を書きます。",
22701 );
22702 runner.state.repo = dir.path().to_owned();
22703 runner.state.candidates[0].worktree = dir.path().to_owned();
22704 let mut author = spec("alpha");
22705 author.command = vec![
22706 "sh".into(),
22707 "-c".into(),
22708 "printf '%s' \"$REWRITE_REPLY\"".into(),
22709 ];
22710 author.env.insert("REWRITE_REPLY".into(), reply.into());
22711 runner.state.config.agents = vec![author];
22712 seed_public_summary(&mut runner);
22713 let winner = runner.state.candidates[0].clone();
22714 let first = runner
22715 .guarded_pr_message(&winner, None, true)
22716 .await
22717 .unwrap();
22718 let message = if accepted {
22719 first.expect("a passing rewrite never asks")
22720 } else {
22721 assert!(first.is_none());
22723 assert!(runner.state.parked);
22724 let store = ask::Questions::open();
22725 let asked: Vec<_> = store
22726 .list()
22727 .into_iter()
22728 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22729 .collect();
22730 assert_eq!(asked.len(), 1);
22731 assert!(!asked[0].detail.contains("secret"));
22732 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22733 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22734 assert_eq!(
22735 asked[0]
22736 .choices
22737 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22738 rewrite.is_some()
22739 );
22740 store
22743 .update(&asked[0].id, |q| {
22744 q.abandon("test");
22745 Ok(())
22746 })
22747 .unwrap();
22748 let again = runner
22749 .guarded_pr_message(&winner, None, true)
22750 .await
22751 .unwrap()
22752 .expect("an abandoned question falls back");
22753 let n = store
22754 .list()
22755 .into_iter()
22756 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22757 .count();
22758 assert_eq!(n, 1, "asked once per rejected text");
22759 again
22760 };
22761 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22762 assert!(!message.body.contains("secret"));
22763 assert!(!message.body.contains("prefix"));
22764 assert!(!message.title.contains("secret"));
22765 assert_eq!(
22766 message.body.contains("Add retries for failed requests."),
22767 accepted
22768 );
22769 assert_eq!(
22770 runner.state.seats["github-text-rewrite"].turns, 1,
22771 "only one rewrite invocation"
22772 );
22773 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22774 if accepted {
22775 assert!(runner.state.github_text.is_none());
22776 assert!(!runner.state.parked);
22777 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22778 assert!(message.body.contains("## Final verification"));
22779 assert!(message.body.contains("Evidence: final diff"));
22780 assert!(
22781 message
22782 .body
22783 .contains(&format!("magi:run/{}", runner.state.id))
22784 );
22785 }
22786 if !accepted {
22787 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22788 assert!(
22789 message
22790 .body
22791 .contains(&format!("magi:run/{}", runner.state.id))
22792 );
22793 }
22794 }
22795 }
22796
22797 fn scripted(id: &str, reply: &str) -> AgentSpec {
22798 let mut a = spec(id);
22799 a.command = vec![
22800 "sh".into(),
22801 "-c".into(),
22802 "printf '%s' \"$REWRITE_REPLY\"".into(),
22803 ];
22804 a.env.insert("REWRITE_REPLY".into(), reply.into());
22805 a
22806 }
22807
22808 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22809 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22810
22811 fn seed_public_summary(runner: &mut Runner) {
22813 let repo = &runner.state.repo;
22814 let g = |args: &[&str]| {
22815 let out = std::process::Command::new("git")
22816 .args(args)
22817 .current_dir(repo)
22818 .quiet()
22819 .output()
22820 .unwrap();
22821 assert!(out.status.success(), "git {args:?}");
22822 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22823 };
22824 g(&["init", "-q", "-b", "main"]);
22825 g(&[
22826 "-c",
22827 "user.name=Test",
22828 "-c",
22829 "user.email=test@example.invalid",
22830 "commit",
22831 "-q",
22832 "--allow-empty",
22833 "-m",
22834 "feat: fixture",
22835 ]);
22836 let head = g(&["rev-parse", "HEAD"]);
22837 g(&["update-ref", "refs/remotes/origin/main", &head]);
22838 let evidence = crate::pr_outcome::summary_evidence(
22839 &runner.state,
22840 crate::pr_outcome::FinalHead::At(&head),
22841 );
22842 let generated = &runner.state.candidates[0].summary;
22843 runner.state.change_summary = Some(crate::run::ChangeSummary {
22844 base: head.clone(),
22845 head,
22846 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22847 title: summary_title(generated).unwrap(),
22848 body: summary_without_title(generated),
22849 });
22850 }
22851
22852 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22853 let mut runner = runner_at(RunStatus::Gating);
22854 runner.state = state_with_summary("add retries", JA_SUMMARY);
22855 if review_only {
22856 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22857 }
22858 runner.state.repo = dir.to_owned();
22859 runner.state.candidates[0].worktree = dir.to_owned();
22860 runner.state.config.agents = agents;
22861 seed_public_summary(&mut runner);
22862 runner
22863 }
22864
22865 fn gate_questions(runner: &Runner) -> usize {
22866 ask::Questions::open()
22867 .list()
22868 .into_iter()
22869 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22870 .count()
22871 }
22872
22873 #[tokio::test]
22874 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22875 crate::run::pin_test_home();
22876 let dir = tempfile::tempdir().unwrap();
22877 let mut runner = rewrite_runner(
22878 dir.path(),
22879 true,
22880 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22881 );
22882 runner.state.config.roles.language_judge =
22883 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22884 let winner = runner.state.candidates[0].clone();
22885 let msg = runner
22886 .guarded_pr_message(&winner, None, true)
22887 .await
22888 .unwrap()
22889 .expect("an English rewrite posts without asking");
22890 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22891 assert!(msg.body.contains("<details>kept</details>"));
22892 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22893 assert_eq!(gate_questions(&runner), 0);
22894 assert!(runner.state.github_text.is_none());
22895 assert!(!runner.state.seats.contains_key("impl-A"));
22896 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22897 }
22898
22899 #[tokio::test]
22900 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22901 crate::run::pin_test_home();
22902 let dir = tempfile::tempdir().unwrap();
22903 let mut runner = rewrite_runner(
22904 dir.path(),
22905 false,
22906 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22907 );
22908 runner.state.config.roles.language_judge =
22909 Some(crate::config::AgentChoice::One("beta".into()));
22910 runner.state.seats.insert(
22911 "impl-A".into(),
22912 SeatState::new("impl-A", "alpha", runner.state.seed),
22913 );
22914 let winner = runner.state.candidates[0].clone();
22915 let msg = runner
22916 .guarded_pr_message(&winner, None, true)
22917 .await
22918 .unwrap()
22919 .expect("rewritten");
22920 assert!(msg.body.contains("<details>kept</details>"));
22921 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22922 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22923 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22924 }
22925
22926 #[tokio::test]
22927 async fn rewrite_reuses_a_resumable_implementer_session() {
22928 crate::run::pin_test_home();
22929 let dir = tempfile::tempdir().unwrap();
22930 let mut runner = rewrite_runner(
22931 dir.path(),
22932 false,
22933 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22934 );
22935 runner.state.config.roles.language_judge =
22936 Some(crate::config::AgentChoice::One("beta".into()));
22937 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22938 seat.turns = 1;
22939 runner.state.seats.insert("impl-A".into(), seat);
22940 let winner = runner.state.candidates[0].clone();
22941 runner
22942 .guarded_pr_message(&winner, None, true)
22943 .await
22944 .unwrap()
22945 .expect("rewritten");
22946 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22947 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22948 }
22949
22950 #[tokio::test]
22951 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22952 crate::run::pin_test_home();
22955 let dir = tempfile::tempdir().unwrap();
22956 let mut exhausted = spec("alpha");
22957 exhausted.command = vec!["false".into()];
22958 let mut runner = rewrite_runner(
22959 dir.path(),
22960 false,
22961 vec![exhausted, scripted("beta", EN_REWRITE)],
22962 );
22963 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22965 "alpha".into(),
22966 "beta".into(),
22967 ]));
22968 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22969 seat.turns = 1;
22970 runner.state.seats.insert("impl-A".into(), seat);
22971 let winner = runner.state.candidates[0].clone();
22972 let msg = runner
22973 .guarded_pr_message(&winner, None, true)
22974 .await
22975 .unwrap()
22976 .expect("the role rewrites after the implementer fails");
22977 assert!(msg.body.contains("`src/net.rs`"));
22978 assert_eq!(
22979 runner.state.seats["impl-A"].turns, 2,
22980 "the implementer was tried once"
22981 );
22982 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22983 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22984 }
22985
22986 #[tokio::test]
22987 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22988 crate::run::pin_test_home();
22989 let dir = tempfile::tempdir().unwrap();
22990 let mut broken = spec("bad");
22991 broken.command = vec!["false".into()];
22992 let mut runner = rewrite_runner(
22993 dir.path(),
22994 true,
22995 vec![
22996 broken,
22997 scripted("noformat", "Add retries."),
22998 scripted("good", EN_REWRITE),
22999 ],
23000 );
23001 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23002 "bad".into(),
23003 "noformat".into(),
23004 "good".into(),
23005 ]));
23006 let winner = runner.state.candidates[0].clone();
23007 let msg = runner
23008 .guarded_pr_message(&winner, None, true)
23009 .await
23010 .unwrap()
23011 .expect("the third agent rewrites");
23012 assert!(msg.body.contains("`src/net.rs`"));
23013 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23014 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23015 }
23016
23017 #[tokio::test]
23018 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23019 crate::run::pin_test_home();
23020 for (agents, role, saved) in [
23022 (
23023 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23024 Some("beta"),
23025 true,
23026 ),
23027 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23028 (Vec::new(), None, false),
23029 ] {
23030 let dir = tempfile::tempdir().unwrap();
23031 let mut runner = rewrite_runner(dir.path(), true, agents);
23032 runner.state.config.roles.language_judge =
23033 role.map(|r| crate::config::AgentChoice::One(r.into()));
23034 let winner = runner.state.candidates[0].clone();
23035 let first = runner
23036 .guarded_pr_message(&winner, None, true)
23037 .await
23038 .unwrap();
23039 assert!(first.is_none(), "withheld text parks the run");
23040 assert!(runner.state.parked);
23041 assert_eq!(gate_questions(&runner), 1);
23042 let g = runner.state.github_text.as_ref().unwrap();
23043 assert_eq!(g.rewrite.is_some(), saved);
23044 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23045 }
23046 }
23047
23048 #[tokio::test]
23049 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23050 crate::run::pin_test_home();
23051 let dir = tempfile::tempdir().unwrap();
23052 let mut runner = runner_at(RunStatus::Gating);
23053 runner.state =
23054 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23055 runner.state.repo = dir.path().to_owned();
23056 runner.state.candidates[0].worktree = dir.path().to_owned();
23057 let mut author = spec("alpha");
23058 author.command = vec![
23059 "sh".into(),
23060 "-c".into(),
23061 "printf '%s' \"$REWRITE_REPLY\"".into(),
23062 ];
23063 author.env.insert(
23064 "REWRITE_REPLY".into(),
23065 format!(
23066 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23067 "English sentence. ".repeat(600)
23068 ),
23069 );
23070 runner.state.config.agents = vec![author];
23071 seed_public_summary(&mut runner);
23072 let winner = runner.state.candidates[0].clone();
23073 assert!(
23074 runner
23075 .guarded_pr_message(&winner, None, true)
23076 .await
23077 .unwrap()
23078 .is_none()
23079 );
23080 let record = runner.state.github_text.clone().unwrap();
23081 let rewrite = record.rewrite.clone().unwrap();
23082 assert_eq!(rewrite.failed, ["body-language"]);
23083 assert!(
23084 rewrite
23085 .body
23086 .ends_with(&format!("magi:run/{}", runner.state.id))
23087 );
23088 let store = ask::Questions::open();
23089 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23090 assert!(
23091 q.choices
23092 .contains(&crate::github_text::USE_REWRITE.to_owned())
23093 );
23094 for label in [
23095 "ORIGINAL",
23096 "FALLBACK",
23097 "REWRITE",
23098 "Rules still failed: body-language",
23099 "truncated",
23100 ] {
23101 assert!(q.detail.contains(label), "{label}");
23102 }
23103 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23104 &runner.state,
23105 &crate::github_text::artifact_name(&record.fingerprint),
23106 ))
23107 .unwrap();
23108 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23109 runner.state =
23111 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23112 store
23113 .update(&q.id, |q| {
23114 q.say("まだ日本語")?;
23115 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23116 })
23117 .unwrap();
23118 assert!(
23119 runner
23120 .guarded_pr_message(&winner, None, true)
23121 .await
23122 .unwrap()
23123 .is_none()
23124 );
23125 let retry = runner.state.github_text.as_ref().unwrap();
23126 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23127 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23128 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23129 assert!(
23130 q.posting_comparison
23131 .as_ref()
23132 .unwrap()
23133 .matches_detail(&q.detail)
23134 );
23135 store
23136 .update(&q.id, |q| {
23137 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23138 })
23139 .unwrap();
23140 let message = runner
23141 .guarded_pr_message(&winner, None, true)
23142 .await
23143 .unwrap()
23144 .unwrap();
23145 assert_eq!(message.title, rewrite.title);
23146 assert_eq!(message.body, rewrite.body);
23147 assert_eq!(
23148 runner.state.github_text.as_ref().unwrap().choice,
23149 Some(GithubTextChoice::Rewrite)
23150 );
23151 let again = runner
23152 .guarded_pr_message(&winner, None, true)
23153 .await
23154 .unwrap()
23155 .unwrap();
23156 assert_eq!(again.title, message.title);
23157 assert_eq!(again.body, message.body);
23158 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23159 assert!(
23160 !runner
23161 .state
23162 .events
23163 .iter()
23164 .any(|e| e.message.contains(&rewrite.title)
23165 || e.message.contains("English sentence."))
23166 );
23167 }
23168
23169 #[tokio::test]
23170 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23171 crate::run::pin_test_home();
23172 let dir = tempfile::tempdir().unwrap();
23173 let mut runner = runner_at(RunStatus::Gating);
23174 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23175 runner.state.repo = dir.path().to_owned();
23176 runner.state.candidates[0].worktree = dir.path().to_owned();
23177 seed_public_summary(&mut runner);
23178 let winner = runner.state.candidates[0].clone();
23179 assert!(
23180 runner
23181 .guarded_pr_message(&winner, None, true)
23182 .await
23183 .unwrap()
23184 .is_none()
23185 );
23186 let record = runner.state.github_text.as_ref().unwrap();
23187 assert!(record.rewrite.is_none());
23188 let mut old = serde_json::to_value(record).unwrap();
23190 old.as_object_mut().unwrap().remove("rewrite");
23191 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23192 assert!(old.rewrite.is_none());
23193 let id = record.question.clone().unwrap();
23194 ask::Questions::open()
23195 .update(&id, |q| {
23196 q.choices.push(crate::github_text::USE_REWRITE.into());
23197 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23198 })
23199 .unwrap();
23200 let message = runner
23201 .guarded_pr_message(&winner, None, true)
23202 .await
23203 .unwrap()
23204 .unwrap();
23205 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23206 assert_eq!(
23207 runner.state.github_text.as_ref().unwrap().choice,
23208 Some(GithubTextChoice::Fallback)
23209 );
23210 }
23211
23212 #[tokio::test]
23213 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23214 crate::run::pin_test_home();
23215 let dir = tempfile::tempdir().unwrap();
23216 let mut runner = runner_at(RunStatus::Gating);
23217 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23218 runner.state.repo = dir.path().to_owned();
23219 runner.state.candidates[0].worktree = dir.path().to_owned();
23220 seed_public_summary(&mut runner);
23221 let winner = runner.state.candidates[0].clone();
23222 assert!(
23223 runner
23224 .guarded_pr_message(&winner, None, true)
23225 .await
23226 .unwrap()
23227 .is_none()
23228 );
23229 let store = ask::Questions::open();
23230 let find = |run: &str| {
23231 store
23232 .list()
23233 .into_iter()
23234 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23235 .unwrap()
23236 };
23237 let run = runner.state.id.clone();
23238 let q = find(&run);
23239 let fp = runner
23240 .state
23241 .github_text
23242 .as_ref()
23243 .unwrap()
23244 .fingerprint
23245 .clone();
23246 let name = crate::github_text::artifact_name(&fp);
23247 let path = crate::run::artifact_path(&runner.state, &name);
23248 assert!(q.detail.contains(&path.display().to_string()));
23249 assert!(q.detail.contains("title-language: title"));
23250 let saved = std::fs::read_to_string(&path).expect("artifact written");
23251 assert!(saved.contains("日本語のタイトル"));
23252 assert!(q.cwd.is_none());
23253 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23254 let say = |id: &str, text: &str, choice: &str| {
23256 store
23257 .update(id, |q| {
23258 q.thread.push(ask::Turn {
23259 who: ask::Who::Operator,
23260 body: text.to_owned(),
23261 at: jiff::Timestamp::now(),
23262 note: None,
23263 });
23264 q.answer(ask::Answer::Choice(choice.to_owned()))
23265 })
23266 .unwrap();
23267 };
23268 say(&q.id, "まだ日本語", "use my text");
23269 assert!(
23270 runner
23271 .guarded_pr_message(&winner, None, true)
23272 .await
23273 .unwrap()
23274 .is_none()
23275 );
23276 let second = find(&run);
23277 assert_ne!(second.id, q.id);
23278 assert!(second.detail.contains("did not pass"));
23279 assert!(second.detail.contains("日本語のタイトル"));
23282 assert!(!second.detail.contains("まだ日本語"));
23283 assert!(second.detail.contains(&path.display().to_string()));
23284 say(&second.id, "fix: retry failed requests", "use my text");
23285 let message = runner
23286 .guarded_pr_message(&winner, None, true)
23287 .await
23288 .unwrap()
23289 .expect("a vetted replacement is posted");
23290 assert_eq!(message.title, "fix: retry failed requests");
23291 assert_eq!(
23292 runner
23293 .state
23294 .github_text
23295 .as_ref()
23296 .unwrap()
23297 .chosen_title
23298 .as_deref(),
23299 Some("fix: retry failed requests")
23300 );
23301 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23302 let again = runner
23304 .guarded_pr_message(&winner, None, true)
23305 .await
23306 .unwrap()
23307 .unwrap();
23308 assert_eq!(again.title, message.title);
23309 assert_eq!(
23310 store
23311 .list()
23312 .into_iter()
23313 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23314 .count(),
23315 2
23316 );
23317 }
23318
23319 #[test]
23320 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23321 let mut state = state_with_summary(
23325 "add retries",
23326 "TITLE: fix(web): batch reads\n- reads run.json once",
23327 );
23328 state.config.graph.language = "ja".to_owned();
23329 let m = pr_message(&state, 'A');
23330 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23331 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23332
23333 let task = "今回やってほしいこと: results projector を直す";
23336 let mut state = state_with_summary(task, "- no title line");
23337 state.config.graph.language = "ja".to_owned();
23338 let m = pr_message(&state, 'A');
23339 assert_eq!(
23340 m.title,
23341 format!("chore: land candidate A of run {}", state.id)
23342 );
23343 assert!(
23344 m.body.contains(&format!(
23345 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23346 )),
23347 "{}",
23348 m.body
23349 );
23350 }
23351
23352 #[test]
23353 fn pr_message_scrubs_home_paths_and_addresses() {
23354 let state = state_with_summary(
23355 "fix it in /Users/someone/src/x",
23356 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23357 );
23358 let m = pr_message(&state, 'A');
23359 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23360 assert!(!m.body.contains(leak), "{}", m.body);
23361 }
23362 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23363 }
23364
23365 #[test]
23366 fn pr_message_survives_a_task_that_closes_details() {
23367 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23368 let m = pr_message(&state, 'A');
23369 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23370 }
23371
23372 #[test]
23373 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23374 let cmd = manual_merge_command(
23375 MergeStyle::Squash,
23376 Path::new("/repo"),
23377 "b",
23378 "fix: \"quoted\" $(x) `y`\n\nbody",
23379 );
23380 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23381 }
23382
23383 #[test]
23384 fn manual_merge_command_matches_the_configured_style() {
23385 let repo = Path::new("/repo");
23386 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23387
23388 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23389 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23390
23391 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23392 assert_eq!(
23393 squash,
23394 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23395 \"Merge magi run 0832 (candidate A)\""
23396 );
23397
23398 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23399 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23400 }
23401
23402 #[test]
23403 fn a_nudge_gets_a_quarter_of_the_budget() {
23404 assert_eq!(retry_budget(secs(1200), true), secs(300));
23406 assert_eq!(retry_budget(secs(3600), true), secs(900));
23407 }
23408
23409 #[test]
23410 fn a_resent_prompt_keeps_the_whole_budget() {
23411 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23414 assert_eq!(retry_budget(secs(60), false), secs(60));
23415 }
23416
23417 #[test]
23418 fn the_floor_never_exceeds_the_original_budget() {
23419 assert_eq!(retry_budget(secs(60), true), secs(60));
23423 assert_eq!(retry_budget(secs(480), true), secs(120));
23424 assert_eq!(retry_budget(secs(0), true), secs(0));
23425 }
23426
23427 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23428 agent::CommandEvidence {
23429 id: "item1".to_owned(),
23430 description: "cargo test".to_owned(),
23431 exit_code,
23432 result_summary: String::new(),
23433 source: "codex".to_owned(),
23434 }
23435 }
23436
23437 #[test]
23438 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23439 assert!(!has_unconfirmed_command(&[]));
23443 }
23444
23445 #[test]
23446 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23447 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23451 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23452 assert!(!has_unconfirmed_command(&[
23453 evidence(Some(0)),
23454 evidence(Some(101))
23455 ]));
23456 }
23457
23458 #[test]
23459 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23460 assert!(has_unconfirmed_command(&[
23461 evidence(Some(0)),
23462 evidence(None)
23463 ]));
23464 }
23465
23466 #[test]
23467 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23468 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23469 assert_eq!(
23470 verified_noop_claim(true, &[], text).as_deref(),
23471 Some("already fixed by b32cfc4, on main.")
23472 );
23473 }
23474
23475 #[test]
23476 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23477 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23480 assert!(verified_noop_claim(false, &[], text).is_none());
23481 }
23482
23483 #[test]
23484 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23485 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23486 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23487 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23489 }
23490
23491 #[test]
23492 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23493 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23494 }
23495
23496 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23499 runner.state.candidates = shape
23500 .iter()
23501 .enumerate()
23502 .map(|(i, &(empty, verified))| Candidate {
23503 index: i,
23504 label: (b'A' + i as u8) as char,
23505 agent: "sonnet".to_owned(),
23506 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23507 worktree: PathBuf::from(format!("/wt/{i}")),
23508 summary: String::new(),
23509 stat: String::new(),
23510 files: 0,
23511 commits: 0,
23512 empty,
23513 failed: None,
23514 verified_noop: verified.map(str::to_owned),
23515 duration_ms: 0,
23516 folded: false,
23517 })
23518 .collect();
23519 }
23520
23521 #[test]
23522 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23523 ask_test_home();
23524 let mut runner = runner_at(RunStatus::Implementing);
23525 set_candidates(
23526 &mut runner,
23527 &[
23528 (true, Some("already on main at b32cfc4")),
23529 (true, Some("same fix, see the existing test")),
23530 ],
23531 );
23532
23533 runner
23534 .after_implement()
23535 .expect("a verified no-op is not an error");
23536
23537 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23538 }
23539
23540 #[test]
23541 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23542 ask_test_home();
23543 let mut runner = runner_at(RunStatus::Implementing);
23544 set_candidates(
23548 &mut runner,
23549 &[(true, Some("already on main at b32cfc4")), (true, None)],
23550 );
23551
23552 let err = runner
23553 .after_implement()
23554 .expect_err("an unverified empty candidate must still fail the run");
23555
23556 assert!(
23557 err.to_string().contains("no candidate produced a change"),
23558 "{err}"
23559 );
23560 assert_eq!(runner.state.status, RunStatus::Failed);
23561 }
23562
23563 #[test]
23564 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23565 ask_test_home();
23566 let mut runner = runner_at(RunStatus::Implementing);
23567 set_candidates(&mut runner, &[(true, None), (true, None)]);
23568
23569 let err = runner
23570 .after_implement()
23571 .expect_err("no candidate declared anything; this is an ordinary failure");
23572
23573 assert!(
23574 err.to_string().contains("no candidate produced a change"),
23575 "{err}"
23576 );
23577 assert_eq!(runner.state.status, RunStatus::Failed);
23578 }
23579}
23580
23581#[derive(Debug)]
23584struct GraftRestoreFailed(String);
23585
23586impl std::fmt::Display for GraftRestoreFailed {
23587 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23588 write!(
23589 f,
23590 "could not restore the winner to its pre-graft commit: {}",
23591 self.0
23592 )
23593 }
23594}
23595
23596impl std::error::Error for GraftRestoreFailed {}
23597
23598#[cfg(test)]
23602mod resource_wait_tests {
23603 use super::*;
23604 use std::collections::BTreeMap;
23605
23606 fn sh(id: &str, script: &str) -> AgentSpec {
23607 AgentSpec {
23608 id: id.to_owned(),
23609 kind: crate::config::AgentKind::Command,
23610 model: None,
23611 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23612 extra_args: Vec::new(),
23613 env: BTreeMap::new(),
23614 prompt_delivery: None,
23615 max_turn_secs: None,
23616 }
23617 }
23618
23619 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23620 SeatJob {
23621 seat: SeatState::new("impl-A", &spec.id, 1),
23622 spec,
23623 cwd: dir.to_path_buf(),
23624 prompt: "do it".to_owned(),
23625 timeout,
23626 allow_write: true,
23627 sessions: false,
23628 artifacts: dir.join("art"),
23629 stem: "impl-A".to_owned(),
23630 handover: None,
23631 }
23632 }
23633
23634 fn state(resource_wait_secs: u64) -> RunState {
23635 let mut config = Config::default();
23636 config.graph.timeout_resource_wait = resource_wait_secs;
23637 RunState::new(
23638 PathBuf::from("/repo"),
23639 "main".to_owned(),
23640 "abc1234".to_owned(),
23641 "task".to_owned(),
23642 config,
23643 )
23644 }
23645
23646 fn hold(cache: &Path) -> crate::cache::Guard {
23647 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23648 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23649 crate::cache::AcquireOutcome::Acquired(g) => g,
23650 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23651 }
23652 }
23653
23654 async fn run(
23655 jobs: Vec<SeatJob>,
23656 sem: Arc<Semaphore>,
23657 cache: Option<&Path>,
23658 st: &mut RunState,
23659 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23660 let prompts = Prompts::default();
23661 let ctx = WaveCtx {
23662 run: "r",
23663 node: "implement",
23664 prompts: &prompts,
23665 cache,
23666 round: None,
23667 carry_seats: false,
23668 pauses: None,
23669 };
23670 wave(jobs, sem, &ctx, st, 0).await
23671 }
23672
23673 #[tokio::test]
23674 async fn verification_cache_wait_preserves_the_command_budget() {
23675 crate::run::pin_test_home();
23676 let tmp = tempfile::tempdir().unwrap();
23677 let cache = tmp.path().join("target");
23678 let guard = hold(&cache);
23679 let release = tokio::spawn(async move {
23680 tokio::time::sleep(Duration::from_millis(100)).await;
23681 guard.release();
23682 });
23683 let mut st = state(10);
23684 let budget = Duration::from_millis(50);
23685 let wait = repository_wait(&st, None);
23686 let (outcomes, _) = with_cache_lease(
23687 &mut st,
23688 Some(&cache),
23689 "gate",
23690 "gate",
23691 tmp.path(),
23692 "abc1234",
23693 budget,
23694 "test gate",
23695 &wait,
23696 |_, available| async move {
23697 assert_eq!(
23698 available, budget,
23699 "contention must not shorten verification"
23700 );
23701 (Vec::new(), false, Vec::new())
23702 },
23703 )
23704 .await
23705 .unwrap();
23706 release.await.unwrap();
23707 assert!(outcomes.is_empty(), "{outcomes:?}");
23708 assert!(
23709 st.events
23710 .iter()
23711 .any(|event| event.message.contains("waiting for"))
23712 );
23713 assert!(st.resource_wait.is_none());
23714 hold(&cache).release();
23716 }
23717
23718 #[tokio::test]
23719 async fn verification_cache_wait_expires_without_starting_commands() {
23720 crate::run::pin_test_home();
23721 let tmp = tempfile::tempdir().unwrap();
23722 let cache = tmp.path().join("target");
23723 let _held = hold(&cache);
23724 let mut st = state(0);
23725 let wait = repository_wait(&st, None);
23726 let (outcomes, _) = with_cache_lease(
23727 &mut st,
23728 Some(&cache),
23729 "gate",
23730 "gate",
23731 tmp.path(),
23732 "abc1234",
23733 Duration::from_secs(60),
23734 "test gate",
23735 &wait,
23736 |_, _| async { panic!("commands must not start without the cache lease") },
23737 )
23738 .await
23739 .unwrap();
23740 assert_eq!(outcomes.len(), 1);
23741 assert!(outcomes[0].resource_blocked);
23742 assert!(st.resource_wait.is_none());
23743 assert!(matches!(
23744 st.resource_block.unwrap().cause,
23745 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23746 ));
23747 }
23748
23749 #[tokio::test]
23750 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23751 crate::run::pin_test_home();
23752 let tmp = tempfile::tempdir().unwrap();
23753 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23754 let cache = tmp.path().join("cache");
23755 let mut state = state(21600);
23756 let wait = repository_wait(&state, None);
23757 let (outcomes, _) = with_cache_lease(
23758 &mut state,
23759 Some(&cache),
23760 "gate",
23761 "gate",
23762 tmp.path(),
23763 "head",
23764 Duration::from_secs(60),
23765 "gate",
23766 &wait,
23767 |_, _| async { panic!("freshness failure must not start verification") },
23768 )
23769 .await
23770 .unwrap();
23771 assert_eq!(outcomes.len(), 1);
23772 assert!(outcomes[0].resource_blocked);
23773 let block = state.resource_block.unwrap();
23774 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23775 assert!(!block.description().contains("timed out"));
23776 }
23777
23778 #[tokio::test]
23779 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23780 crate::run::pin_test_home();
23781 let tmp = tempfile::tempdir().unwrap();
23782 let cache = tmp.path().join("cache");
23783 let _held = hold(&cache);
23784 let mut state = state(21600);
23785 let pause = Pause::new();
23786 let interrupt = Pause::new();
23787 let cause = ParkCause::Operator;
23788 pause.park_for(cause.clone());
23789 let prompts = Prompts::default();
23790 let ctx = WaveCtx {
23791 run: "test",
23792 node: "fix",
23793 prompts: &prompts,
23794 cache: Some(&cache),
23795 round: None,
23796 carry_seats: false,
23797 pauses: Some((&pause, &interrupt)),
23798 };
23799 let result = wave(
23800 vec![job(
23801 sh("writer", "exit 99"),
23802 tmp.path(),
23803 Duration::from_secs(1),
23804 )],
23805 Arc::new(Semaphore::new(1)),
23806 &ctx,
23807 &mut state,
23808 0,
23809 )
23810 .await;
23811 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23812 assert_eq!(result[0].1.turns, 0);
23813 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23814 assert_eq!(
23815 state.resource_block.unwrap().cause,
23816 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23817 );
23818 assert!(state.active.is_empty());
23819 }
23820
23821 #[tokio::test]
23822 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23823 crate::run::pin_test_home();
23824 let tmp = tempfile::tempdir().unwrap();
23825 let cache = tmp.path().join("target-a");
23826 let guard = hold(&cache);
23827 tokio::spawn(async move {
23829 tokio::time::sleep(Duration::from_millis(1500)).await;
23830 guard.release();
23831 });
23832 let mut st = state(60);
23833 let results = run(
23834 vec![job(
23835 sh("alpha", "echo ok"),
23836 tmp.path(),
23837 Duration::from_secs(1),
23838 )],
23839 Arc::new(Semaphore::new(1)),
23840 Some(&cache),
23841 &mut st,
23842 )
23843 .await;
23844 assert!(
23845 matches!(results[0].2, AgentOutcome::Ok(_)),
23846 "the wait must not be charged to the agent"
23847 );
23848 assert!(st.resource_block.is_none());
23849 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23850 assert!(st.resource_wait.is_none() && st.active.is_empty());
23851 }
23852
23853 #[tokio::test]
23854 #[cfg(unix)]
23855 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23856 use std::os::fd::AsRawFd as _;
23857 crate::run::pin_test_home();
23858 let tmp = tempfile::tempdir().unwrap();
23859 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23860 let common = tmp.path().join(".git");
23861 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23862 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23863 assert_eq!(
23864 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23865 0
23866 );
23867 let mut st = state(1);
23868 st.config.coordination = crate::config::test_coordination();
23869 let mut reader = job(
23870 sh("reader", "echo must-not-run"),
23871 tmp.path(),
23872 Duration::from_secs(30),
23873 );
23874 reader.allow_write = false;
23875 let results = run(
23876 vec![reader.clone()],
23877 Arc::new(Semaphore::new(1)),
23878 None,
23879 &mut st,
23880 )
23881 .await;
23882 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23883 assert_eq!(results[0].1.turns, 0);
23884 assert!(FailClass::of(&results[0].2).is_none());
23885 assert_eq!(
23886 st.resource_block.as_ref().unwrap().kind,
23887 "repository coordination"
23888 );
23889 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23890 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23891 drop(external);
23892 let editor = crate::repo_lock::SeatLease::for_checkout(
23893 tmp.path(),
23894 true,
23895 crate::config::test_coordination().as_ref(),
23896 )
23897 .await
23898 .unwrap();
23899 let pause = Pause::new();
23900 let interrupt = Pause::new();
23901 let prompts = Prompts::default();
23902 let ctx = WaveCtx {
23903 run: "r",
23904 node: "review",
23905 prompts: &prompts,
23906 cache: None,
23907 round: None,
23908 carry_seats: false,
23909 pauses: Some((&pause, &interrupt)),
23910 };
23911 let (results, ()) = tokio::join!(
23912 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23913 async {
23914 tokio::time::sleep(Duration::from_millis(50)).await;
23915 pause.park();
23916 }
23917 );
23918 assert!(
23919 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23920 );
23921 assert_eq!(results[0].1.turns, 0);
23922 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23923 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23924 drop(editor);
23925 }
23926
23927 #[tokio::test]
23928 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23929 crate::run::pin_test_home();
23930 let tmp = tempfile::tempdir().unwrap();
23931 let cache = tmp.path().join("target-b");
23932 let _held = hold(&cache);
23933 let mut st = state(1);
23934 let results = run(
23935 vec![job(
23936 sh("alpha", "echo ok"),
23937 tmp.path(),
23938 Duration::from_secs(1),
23939 )],
23940 Arc::new(Semaphore::new(1)),
23941 Some(&cache),
23942 &mut st,
23943 )
23944 .await;
23945 let (_, seat, out) = &results[0];
23946 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23947 assert!(wave_blocked(&results));
23948 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23949 assert_eq!(seat.turns, 0, "nothing was asked");
23950 assert!(
23951 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23952 "no agent was spawned"
23953 );
23954 assert!(FailClass::of(out).is_none(), "not a failure class");
23955 let block = st.resource_block.as_ref().expect("recorded for the resume");
23956 assert_eq!(block.kind, "shared build cache");
23957 assert!(matches!(
23958 block.cause,
23959 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23960 ));
23961 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23962 assert!(st.active.is_empty());
23963 }
23964
23965 #[tokio::test]
23966 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23967 crate::run::pin_test_home();
23968 let tmp = tempfile::tempdir().unwrap();
23969 let mut st = state(1);
23970 let results = run(
23971 vec![job(
23972 sh("alpha", "echo ok"),
23973 tmp.path(),
23974 Duration::from_secs(1),
23975 )],
23976 Arc::new(Semaphore::new(0)),
23977 None,
23978 &mut st,
23979 )
23980 .await;
23981 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23982 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23983 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23984 }
23985
23986 #[tokio::test]
23987 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23988 crate::run::pin_test_home();
23989 let tmp = tempfile::tempdir().unwrap();
23990 let mut st = state(60);
23991 let results = run(
23992 vec![job(
23993 sh("alpha", "sleep 30"),
23994 tmp.path(),
23995 Duration::from_secs(1),
23996 )],
23997 Arc::new(Semaphore::new(1)),
23998 None,
23999 &mut st,
24000 )
24001 .await;
24002 let out = &results[0].2;
24003 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24004 let class = FailClass::of(out).expect("a failure");
24005 assert_eq!(class, FailClass::Timeout);
24006 assert!(should_hand_over(None, &class), "existing handover policy");
24007 assert!(st.resource_block.is_none());
24008 }
24009}