1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false, None).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true, None).await
1357 }
1358
1359 pub(crate) async fn authorize_base_recovery_control(
1360 home: &Path,
1361 task: &str,
1362 control: &mut crate::base_recovery_control::Transaction,
1363 ) -> Result<()> {
1364 let id = control.request.run.clone();
1365 Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366 }
1367
1368 async fn authorize_base_recovery_request(
1369 home: &Path,
1370 id: &str,
1371 task: &str,
1372 resumption: bool,
1373 mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374 ) -> Result<()> {
1375 let _owner = crate::pr_head::owner_lock(home, id)?;
1376 let mut state = RunState::load_under(id, home)?;
1377 crate::pr_head::ensure_owner(&state)?;
1378 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379 anyhow::ensure!(
1380 queued.execution_runs().last().is_some_and(|run| run == id)
1381 && queued.status != crate::queue::TaskStatus::Done
1382 && !queued.fresh_start
1383 && queued.review_branch.is_none(),
1384 "run task ownership changed"
1385 );
1386 anyhow::ensure!(
1387 state
1388 .origin
1389 .as_ref()
1390 .and_then(|o| o.task.as_deref())
1391 .is_none_or(|owner| owner == task),
1392 "run belongs to another task"
1393 );
1394 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395 anyhow::ensure!(
1396 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397 "run is live or its ownership is unknown"
1398 );
1399 let _execution = crate::driver::maintenance(home, id)?;
1402 anyhow::ensure!(
1403 state.status.resumable()
1404 && state.released_to.is_none()
1405 && state.retired_to.is_none()
1406 && state.pr_taken_over_by.is_none(),
1407 "run ownership is final"
1408 );
1409 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410 anyhow::ensure!(
1411 !state.pr_publication_pending
1412 && !crate::handover::landing_checkpoint(&state)
1413 && state.merge_decision.is_none(),
1414 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415 );
1416 let winner = state.winner().context("run has no retained winner")?;
1417 anyhow::ensure!(
1418 git::is_clean(&winner.worktree).await?,
1419 "retained worktree has uncommitted changes"
1420 );
1421 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422 anyhow::ensure!(
1423 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424 "retained worktree does not match the winner branch"
1425 );
1426 if let Some(control) = control.as_ref() {
1427 anyhow::ensure!(
1428 head.starts_with(&control.request.head),
1429 "retained HEAD changed since the owner request"
1430 );
1431 anyhow::ensure!(
1432 state
1433 .base_sync_recovery
1434 .stop
1435 .as_ref()
1436 .is_none_or(|stop| stop.head == head),
1437 "retained HEAD differs from the recorded stop head; inspect it first"
1438 );
1439 }
1440 state.ensure_base_episode();
1441 state.record_base_sync_stop(&head);
1442 anyhow::ensure!(
1443 state
1444 .base_sync_recovery
1445 .stop
1446 .as_ref()
1447 .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448 "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449 );
1450 let fetched =
1451 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452 anyhow::ensure!(
1453 fetched.ok(),
1454 "cannot authorize base recovery without a fetched base"
1455 );
1456 let approved_tip = git::rev_parse(
1457 &state.repo,
1458 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459 )
1460 .await?;
1461 if let Some(control) = control.as_deref_mut() {
1462 let queue = crate::queue::Queue::at(home.join("queue"));
1463 control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464 if let Some(prior) = state
1465 .base_sync_recovery
1466 .authorizations
1467 .iter()
1468 .find(|a| a.request.as_deref() == Some(&control.key()))
1469 {
1470 anyhow::ensure!(
1471 !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472 "recovery request was consumed or invalidated; it cannot queue again"
1473 );
1474 return control.enqueue(&queue);
1475 }
1476 }
1477 if resumption {
1478 anyhow::ensure!(
1479 crate::daemon::compatible_run_instruction(&state, &queued),
1480 "resumption instruction does not match the retained run"
1481 );
1482 let approval = queued
1483 .resumption
1484 .as_ref()
1485 .context("no operator resumption")?;
1486 anyhow::ensure!(
1487 approval.instruction == queued.instruction,
1488 "resumption approval does not match the current task instruction"
1489 );
1490 state.authorize_base_sync_resumption(task, approval)?;
1491 } else {
1492 state.authorize_base_sync(task)?;
1493 }
1494 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495 state
1496 .base_sync_recovery
1497 .authorizations
1498 .last_mut()
1499 .unwrap()
1500 .scope = Some(crate::base_sync_recovery::Scope {
1501 tip: approved_tip,
1502 instruction: queued.instruction.clone(),
1503 episode,
1504 });
1505 if let Some(control) = control.as_deref_mut() {
1506 let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507 authorization.request = Some(control.key());
1508 authorization.generation = control.generation();
1509 authorization.decision = control.decision();
1510 authorization.question = control
1511 .request
1512 .decision
1513 .as_ref()
1514 .map(|_| control.request.request.clone());
1515 }
1516 state.event(
1517 "base-recovery",
1518 "owner authorized a bounded base synchronization interval; fresh verification required",
1519 );
1520 state.save_under(home)?;
1521 if let Some(control) = control {
1522 control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523 }
1524 Ok(())
1525 }
1526
1527 pub fn resume(id: &str) -> Result<Self> {
1529 let state = RunState::load(id)?;
1530 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531 if state.startup.as_ref().is_some_and(|s| !s.complete)
1532 || (state.status == RunStatus::Prep
1533 && state.instruction.trim().is_empty()
1534 && state.candidates.is_empty()
1535 && state.prep_complete == Some(false))
1536 {
1537 bail!(
1538 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539 );
1540 }
1541 crate::pr_head::ensure_owner(&state)?;
1542 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543 bail!(
1544 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545 );
1546 }
1547 if let Some(to) = &state.released_to {
1548 bail!(
1549 "run {} cannot be resumed: its worktree was released to run {}",
1550 state.short(),
1551 crate::run::short_of(to)
1552 );
1553 }
1554 let roles = state.config.resolve_roles()?;
1555 if let Some(id) = &state.config.graph.selected_implementer
1556 && state.candidates.iter().any(|c| c.agent != *id)
1557 {
1558 bail!(
1559 "replacement implementer pin does not match the saved candidates; task must stay held"
1560 );
1561 }
1562 let max_parallel = state.config.graph.max_parallel.max(1);
1563 Ok(Self {
1564 state,
1565 roles,
1566 sem: Arc::new(Semaphore::new(max_parallel)),
1567 pause: Pause::new(),
1568 interrupt: Pause::new(),
1569 })
1570 }
1571
1572 pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579 Box::pin(self.execute_inner())
1582 }
1583
1584 async fn execute_inner(&mut self) -> Result<()> {
1585 anyhow::ensure!(
1586 self.state.startup.as_ref().is_none_or(|s| s.complete),
1587 "startup incomplete; inspect retained evidence before explicit recovery"
1588 );
1589 if let Some(selection) = &self.state.pr_recovery
1590 && (selection.source_proof.is_some() || selection.source_run.is_some())
1591 {
1592 Box::pin(selection.verify_source(&self.state.repo)).await?;
1593 }
1594 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597 || self.state.land_approval.is_some()
1598 || self.state.merge_decision.is_some()
1599 || !self.state.merge_repairs.is_empty()
1600 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601 {
1602 let claim = FixClaim::acquire(&self.state.dir())?;
1603 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604 let disk = RunState::load(&self.state.id)?;
1605 let checkpoint = |s: &RunState| {
1606 (
1607 s.merge_repairs.len(),
1608 s.merge_repairs
1609 .last()
1610 .map(|r| (r.question.clone(), r.phase)),
1611 s.land_approval
1612 .as_ref()
1613 .map(|a| (a.question.clone(), a.head.clone())),
1614 )
1615 };
1616 if checkpoint(&disk) != checkpoint(&self.state) {
1617 bail!(
1618 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619 );
1620 }
1621 Some(claim)
1622 } else {
1623 None
1624 };
1625 let mut execution = self.claim_driver().await?;
1626 let runner = &mut *execution.runner;
1627 let wait = runner.repository_wait();
1628 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631 let result = match result {
1632 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633 record_repository_block(&mut runner.state, &e);
1634 runner.park_here()?;
1635 Ok(())
1636 }
1637 other => other,
1638 };
1639 execution.driver.finish(&mut runner.state).await?;
1640 let ended = if result.is_err() {
1641 Some({
1642 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643 if let Err(error) = &result {
1644 notice.message = format!(
1645 "Run {} stopped with an error: {}",
1646 runner.state.id,
1647 error.to_string().lines().next().unwrap_or("unknown error")
1648 );
1649 }
1650 notice
1651 })
1652 } else {
1653 crate::notices::run_ended(&runner.state)
1654 };
1655 if let Some(notice) = ended {
1656 crate::notices::raise(notice);
1657 }
1658 result
1659 }
1660
1661 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663 }
1664
1665 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666 let wait = self.repository_wait();
1667 coordinate_operation(&mut self.state, cwd, &wait).await
1668 }
1669
1670 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672 let owner = match crate::pr_head::owner_lock_wait(
1673 &crate::run::home(),
1674 &self.state.id,
1675 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676 )
1677 .await
1678 {
1679 Ok(owner) => owner,
1680 Err(error) => {
1681 let detail = format!("worktree maintenance or ownership update: {error:#}");
1682 self.state.resource_block = Some(crate::run::ResourceBlock {
1686 cause: crate::run::ResourceBlockCause::Other,
1687 node: "start".into(),
1688 kind: "worktree maintenance".into(),
1689 detail: detail.clone(),
1690 at: Timestamp::now(),
1691 });
1692 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693 }
1694 };
1695 let disk = RunState::load(&self.state.id)?;
1696 crate::pr_head::ensure_owner(&disk)?;
1697 let claimed =
1698 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699 anyhow::ensure!(
1704 disk.liveness(claimed) != crate::run::Liveness::Live,
1705 "run already has a live driver; wait for its execution to end"
1706 );
1707 if let Some(to) = &disk.released_to {
1708 bail!(
1709 "run {} cannot continue: its worktree was released to run {}",
1710 self.state.short(),
1711 crate::run::short_of(to)
1712 );
1713 }
1714 anyhow::ensure!(
1715 serde_json::to_value(&disk.base_sync_recovery)?
1716 == serde_json::to_value(&self.state.base_sync_recovery)?,
1717 "base recovery changed since this run was loaded; reload the saved run"
1718 );
1719 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720 let mut execution = DriverExecution {
1721 runner: self,
1722 driver,
1723 };
1724 let _owner = owner;
1727 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728 .await?;
1729 let state = &mut execution.runner.state;
1730 state.parked = false;
1735 state.resource_wait = None;
1736 state.resource_block = None;
1737 state.clear_active();
1744 let pid = std::process::id();
1759 state.driver_pid = Some(pid);
1760 state.driver_started_at = crate::proc::process_started_at(pid);
1761 state.driver_exited = false;
1762 state.driver_claimed = Some(true);
1763 state.begin_pass();
1764 execution.driver.bind(state);
1765 state.save()?;
1766 execution.driver.record(state)?;
1767 Ok(execution)
1768 }
1769
1770 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774 if let Err(error) = &result
1775 && !error.is::<crate::repo_lock::ResourceBlocked>()
1776 && self
1777 .state
1778 .repair_grants
1779 .last()
1780 .is_some_and(|g| g.conflict.is_none())
1781 {
1782 self.state.repair_grants.last_mut().unwrap().phase =
1783 crate::repair_budget::Phase::Complete;
1784 self.state.status = RunStatus::Blocked;
1785 self.state.save()?;
1786 bail!(
1787 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788 self.state.passes.last().map_or(0, |p| p.index)
1789 );
1790 }
1791 result
1792 }
1793
1794 async fn execute_granted_repair(&mut self) -> Result<()> {
1797 use crate::repair_budget::{Kind, Phase};
1798 let index = self.state.repair_grants.len() - 1;
1799 let winner = self
1800 .state
1801 .winner()
1802 .cloned()
1803 .context("repair winner missing")?;
1804 loop {
1805 let grant = self.state.repair_grants[index].clone();
1806 match grant.phase {
1807 Phase::Pending => {
1808 if grant.spent >= grant.receipt.request.rounds {
1809 self.state.repair_grants[index].phase = Phase::Complete;
1810 self.state.status = RunStatus::Blocked;
1811 self.state.save()?;
1812 return Ok(());
1813 }
1814 self.state.repair_grants[index].spent += 1;
1817 self.state.repair_grants[index].phase = Phase::Fixing;
1818 self.state.save()?;
1819 match grant.receipt.request.kind {
1820 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821 Kind::Gate => {
1822 let failed = self.state.gate.clone();
1823 let _ = self.gate_fix_round(&winner, &failed).await?;
1824 }
1825 Kind::Review => {
1826 let mut record = self
1827 .state
1828 .reviews
1829 .last()
1830 .cloned()
1831 .context("review failure missing")?;
1832 let round = record.round;
1833 let step = self.run_fix_round(&winner, &mut record).await?;
1834 self.state.reviews[round - 1] = record;
1835 if matches!(step, FixStep::Recorded { report_lost: true }) {
1836 self.state.repair_grants[index].phase = Phase::Complete;
1837 self.state.status = RunStatus::Blocked;
1838 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839 self.state.save()?;
1840 return Ok(());
1841 }
1842 if matches!(step, FixStep::Resource) {
1843 self.stop_for_resource()?;
1844 return Ok(());
1845 }
1846 self.state.save()?;
1847 }
1848 }
1849 if self.resource_halt()? {
1850 return Ok(());
1851 }
1852 }
1853 Phase::Fixing => {
1854 if grant.receipt.request.kind == Kind::BaseConflict {
1855 self.repair_base_conflict(false).await?;
1856 }
1857 let _coordination = self.coordinate(&winner.worktree).await?;
1860 let rescued = git::rescue_commit(
1861 &winner.worktree,
1862 "magi: preserve interrupted granted repair",
1863 )
1864 .await?;
1865 self.state.note_withheld("repair-grant", &rescued.withheld);
1866 self.state
1869 .verification_history
1870 .push(crate::run::VerificationEvidence {
1871 superseded_by: None,
1872 archived_at: Some(Timestamp::now()),
1873 fixes_spent: Some(self.state.gate_fixes.len()),
1874 gate_head: self.state.gate_head.take(),
1875 gate_ran: std::mem::take(&mut self.state.gate_ran),
1876 gate: std::mem::take(&mut self.state.gate),
1877 e2e: self.state.head_e2e.take(),
1878 });
1879 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880 self.state
1881 .verification_history
1882 .push(crate::run::VerificationEvidence {
1883 superseded_by: None,
1884 archived_at: Some(Timestamp::now()),
1885 fixes_spent: Some(self.state.gate_fixes.len()),
1886 gate_head: None,
1887 gate_ran: false,
1888 gate: Vec::new(),
1889 e2e: Some(e2e),
1890 });
1891 }
1892 self.state.repair_grants[index].review_start = self.state.reviews.len();
1893 self.state.repair_grants[index].head =
1894 git::rev_parse(&winner.worktree, "HEAD").await?;
1895 self.state.repair_grants[index].phase = Phase::Reviewing;
1896 if grant.receipt.request.kind == Kind::BaseConflict {
1897 self.state.base_sync_recovery.stop = None;
1898 if let Some(sync) = &mut self.state.base_sync {
1899 sync.tip = grant
1900 .receipt
1901 .request
1902 .tip
1903 .clone()
1904 .context("repair base tip missing")?;
1905 sync.conflict = None;
1906 sync.behind = 0;
1907 }
1908 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909 }
1910 self.state.save()?;
1911 }
1912 Phase::Reviewing => {
1913 self.review_loop_pass().await?;
1914 if self.resource_halt()? {
1915 return Ok(());
1916 }
1917 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918 let clean = self.state.reviews.len() > grant.review_start
1919 && self.state.reviews.last().is_some_and(|r| {
1920 r.head == head
1921 && r.clean
1922 && r.expected > 0
1923 && r.answered == r.expected
1924 && matches!(
1925 r.e2e_status(),
1926 E2eStatus::Passed | E2eStatus::NotConfigured
1927 )
1928 });
1929 if !clean {
1930 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932 == Kind::Review
1933 && grant.spent < grant.receipt.request.rounds
1934 && self.state.reviews.last().is_some_and(|r| {
1935 r.fix.is_none()
1936 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937 }) {
1938 Phase::Pending
1939 } else {
1940 Phase::Complete
1941 };
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "repair-grant",
1945 "fresh review did not clear the repaired head; no publication",
1946 );
1947 self.state.save()?;
1948 if self.state.repair_grants[index].phase == Phase::Pending {
1949 continue;
1950 }
1951 return Ok(());
1952 }
1953 self.state.repair_grants[index].phase = Phase::Gating;
1954 self.state.save()?;
1955 }
1956 Phase::Gating => {
1957 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958 self.state.reconcile_verification_head(&head);
1959 let wait = self.repository_wait();
1960 if self
1961 .state
1962 .reviews
1963 .last()
1964 .is_none_or(|r| r.head != head || !r.clean)
1965 {
1966 self.state.status = RunStatus::Blocked;
1967 self.state.repair_grants[index].phase = Phase::Complete;
1968 self.state.event(
1969 "repair-grant",
1970 "head changed since fresh review; repair held",
1971 );
1972 self.state.save()?;
1973 return Ok(());
1974 }
1975 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976 return Ok(());
1977 }
1978 let outcomes = self.run_gate(&winner).await?;
1979 if verify_inconclusive(&outcomes) {
1980 return Ok(());
1981 }
1982 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983 if after != head {
1984 self.state.repair_grants[index].review_start = self.state.reviews.len();
1985 self.state.repair_grants[index].head = after;
1986 self.state.repair_grants[index].phase = Phase::Reviewing;
1987 self.state.status = RunStatus::Blocked;
1988 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let passed = outcomes.iter().all(CommandOutcome::ok);
1993 if passed
1996 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997 {
1998 return Ok(());
1999 }
2000 self.state.repair_grants[index].succeeded = passed;
2001 self.state.gate = outcomes;
2002 self.state.gate_ran = true;
2003 self.state.status = if passed {
2004 RunStatus::Gating
2005 } else {
2006 RunStatus::Blocked
2007 };
2008 self.state.repair_grants[index].phase = if !passed
2009 && grant.receipt.request.kind == Kind::Gate
2010 && grant.spent < grant.receipt.request.rounds
2011 {
2012 Phase::Pending
2013 } else {
2014 Phase::Complete
2015 };
2016 if passed && self.state.config.merge.mode == MergeMode::Pr {
2017 self.state.pr_publication_pending = true;
2019 } else if passed {
2020 self.state.status = RunStatus::Ready;
2021 }
2022 self.state.save()?;
2023 if !passed {
2024 continue;
2025 }
2026 if self.state.config.merge.mode != MergeMode::Pr {
2030 self.state.status = RunStatus::Ready;
2031 self.state.event(
2032 "repair-grant",
2033 "repair verified locally; grant carries no merge authorization",
2034 );
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 return self.merge().await;
2039 }
2040 Phase::Complete => return Ok(()),
2041 }
2042 }
2043 }
2044
2045 async fn execute_graph(&mut self) -> Result<()> {
2046 let archive_home = crate::run::home().canonicalize()?;
2050 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051 let repo = self.state.repo.clone();
2052 let _coordination = self.coordinate(&repo).await?;
2053 ensure_setup_config(&mut self.state, &repo).await?;
2054 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055 }
2056 self.state.configure_recovery_resume()?;
2057 if self
2058 .state
2059 .repair_grants
2060 .last()
2061 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062 {
2063 return self.execute_granted_repair().await;
2064 }
2065 if self
2066 .state
2067 .repair_grants
2068 .last()
2069 .is_some_and(|g| !g.succeeded)
2070 {
2071 self.state.status = RunStatus::Blocked;
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.persist_candidates().await?;
2078 if self.state.base_sync_blocked() {
2081 self.sync_to_base().await?;
2082 if !self.state.base_sync_blocked()
2083 && self.state.status != RunStatus::AlreadyInBase
2084 && !self.resource_halt()?
2085 {
2086 self.finish_after_tally().await?;
2087 }
2088 return Ok(());
2089 }
2090 if self
2091 .state
2092 .merge_decision
2093 .as_ref()
2094 .is_some_and(|decision| !decision.closed)
2095 && self.state.status != RunStatus::Landing
2096 {
2097 if crate::merge_decision::resumable_hold(&self.state) {
2098 self.state.merge = Some(MergeOutcome {
2099 mode: MergeMode::Pr,
2100 ok: false,
2101 detail: self
2102 .state
2103 .merge_decision
2104 .as_ref()
2105 .unwrap()
2106 .target
2107 .pr
2108 .clone(),
2109 empty: false,
2110 });
2111 self.run_land().await?;
2112 self.settle_questions();
2113 return Ok(());
2114 }
2115 bail!(
2116 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117 );
2118 }
2119 if self.state.config.graph.selected_implementer.is_some()
2120 && self.state.status == RunStatus::Blocked
2121 && self.state.tally.is_none()
2122 && self.state.candidate_preservation.is_empty()
2123 && self.state.interrupted_implementations.is_empty()
2124 {
2125 bail!(
2126 "selected implementation is held; explicitly release the owning task to retry it"
2127 );
2128 }
2129
2130 if self.state.merge_repairs.last().is_some_and(|r| {
2131 matches!(
2132 r.phase,
2133 crate::run::MergeRepairPhase::Requested
2134 | crate::run::MergeRepairPhase::Fixing
2135 | crate::run::MergeRepairPhase::Reviewing
2136 )
2137 }) {
2138 if self.repair_merge_review().await? {
2139 self.run_land().await?;
2140 }
2141 self.settle_questions();
2142 return Ok(());
2143 }
2144 if self.state.pr_publication_pending {
2147 self.merge().await?;
2148 if self.resource_halt()? {
2149 return Ok(());
2150 }
2151 self.state.save()?;
2152 self.settle_questions();
2153 return Ok(());
2154 }
2155 if self.state.status == RunStatus::Stalled {
2168 if self.recover_stall().await? {
2169 self.finish_after_tally().await?;
2170 } else {
2171 self.state.save()?;
2173 }
2174 return Ok(());
2175 }
2176 if self.state.status == RunStatus::Landing
2186 || crate::handover::landing_checkpoint(&self.state)
2187 {
2188 self.state.land_sync_pending = true;
2189 self.state.save()?;
2190 self.run_land().await?;
2191 self.settle_questions();
2196 return Ok(());
2197 }
2198 if self.state.github_text.is_some()
2203 && self.state.merge.is_none()
2204 && self.state.status == RunStatus::Gating
2205 {
2206 self.merge().await?;
2207 if self.resource_halt()? {
2208 return Ok(());
2209 }
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213 if let Err(e) = self.prep().await {
2214 if e.is::<crate::repo_lock::ResourceBlocked>() {
2215 return Err(e);
2216 }
2217 self.state.status = RunStatus::Blocked;
2218 self.state
2219 .prep_failure
2220 .get_or_insert_with(|| format!("{e:#}"));
2221 self.state
2222 .event("prep", format!("preparation stopped: {e:#}"));
2223 self.state.save()?;
2224 return Err(e);
2225 }
2226 if self.park_here()? {
2227 return Ok(());
2228 }
2229 self.advise().await?;
2230 if self.park_here()? {
2231 return Ok(());
2232 }
2233 self.implement().await?;
2234 if self.park_here()? {
2235 return Ok(());
2236 }
2237 if self.state.status == RunStatus::VerifiedNoop {
2241 return Ok(());
2242 }
2243 self.judge().await?;
2244 if self.park_here()? {
2245 return Ok(());
2246 }
2247 self.deliberate().await?;
2248 if self.park_here()? {
2249 return Ok(());
2250 }
2251 self.vote().await?;
2252 if self.park_here()? {
2253 return Ok(());
2254 }
2255 self.tally()?;
2256 if self.state.status == RunStatus::Stalled {
2261 self.state.save()?;
2265 return Ok(());
2266 }
2267 self.finish_after_tally().await?;
2268 Ok(())
2269 }
2270
2271 fn stop_for_resource(&mut self) -> Result<()> {
2276 if self.pause.parked() || self.interrupt.parked() {
2279 self.park_here()?;
2280 return Ok(());
2281 }
2282 if self.state.resource_block.is_some() {
2283 self.state.status = RunStatus::Blocked;
2284 }
2285 self.state.save()
2286 }
2287
2288 fn resource_halt(&mut self) -> Result<bool> {
2290 if self.state.resource_block.is_none() {
2291 return Ok(false);
2292 }
2293 self.stop_for_resource()?;
2294 Ok(true)
2295 }
2296
2297 fn park_here(&mut self) -> Result<bool> {
2304 let parking = self.pause.parked() || self.interrupt.parked();
2305 if !parking && self.state.resource_block.is_some() {
2306 self.stop_for_resource()?;
2307 return Ok(true);
2308 }
2309 if !self.pause.parked() && !self.interrupt.parked() {
2314 return Ok(false);
2315 }
2316 if self.state.parked {
2317 return Ok(true);
2318 }
2319 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320 if let Some(block) = &mut self.state.resource_block
2321 && matches!(
2322 block.cause,
2323 crate::run::ResourceBlockCause::Cancelled { .. }
2324 )
2325 {
2326 block.cause = crate::run::ResourceBlockCause::Cancelled {
2327 park: cause.clone(),
2328 };
2329 }
2330 let why = match cause.as_ref().map(ParkCause::label) {
2331 Some(reason) => format!(
2332 "parked after `{}` ({reason}) — resume to carry on from here",
2333 self.state.status.as_str()
2334 ),
2335 None => format!(
2336 "parked after `{}` — resume to carry on from here",
2337 self.state.status.as_str()
2338 ),
2339 };
2340 self.state.event("park", why);
2341 self.state
2342 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343 self.state.parked = true;
2344 self.state.save()?;
2345 Ok(true)
2346 }
2347
2348 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350 if self.state.prep_failure.is_none() {
2351 return Ok(());
2352 }
2353 let failed_at = self
2354 .state
2355 .events
2356 .iter()
2357 .rev()
2358 .find(|event| event.node == "prep")
2359 .map_or(self.state.updated_at, |event| event.at);
2360 anyhow::ensure!(
2361 approval_at > failed_at,
2362 "preparation retry requires a new explicit release after the failure"
2363 );
2364 self.state.prep_failure = None;
2365 self.state.prep_complete = Some(false);
2366 self.state
2367 .event("prep-retry", "operator approved retry of preparation");
2368 Ok(())
2369 }
2370
2371 pub fn on_pause(&mut self, pause: Pause) {
2373 self.pause = pause;
2374 }
2375
2376 pub fn watch_interrupt(&mut self, pause: Pause) {
2382 self.interrupt = pause;
2383 }
2384
2385 fn settle_questions(&mut self) {
2405 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407 }
2408 }
2409
2410 fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413 Box::pin(self.finish_after_tally_inner())
2414 }
2415
2416 async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417 self.graft().await?;
2420 if self.resource_halt()? {
2421 return Ok(());
2422 }
2423 self.fold_losers().await?;
2424 self.sync_to_base().await?;
2429 if self.hold_base_sync().await? || self.resource_halt()? {
2430 return Ok(());
2431 }
2432 if self.state.status == RunStatus::AlreadyInBase {
2433 return Ok(());
2434 }
2435 self.review_loop().await?;
2436 if self.resource_halt()? {
2437 return Ok(());
2438 }
2439 self.sync_to_base().await?;
2440 if self.hold_base_sync().await? || self.resource_halt()? {
2441 return Ok(());
2442 }
2443 if self.state.status == RunStatus::AlreadyInBase {
2444 return Ok(());
2445 }
2446 self.gate().await?;
2447 self.merge().await?;
2448 if self.resource_halt()? {
2449 return Ok(());
2450 }
2451 self.state.save()?;
2452 Ok(())
2453 }
2454
2455 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462 let result = match ensure_setup_config(&mut self.state, repo).await {
2463 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464 Err(e) => Err(e),
2465 };
2466 if let Err(e) = result {
2467 git::worktree_remove_clean(repo, wt).await.ok();
2468 if in_prep {
2469 self.rollback_prep(repo).await;
2470 }
2471 self.state.status = RunStatus::Blocked;
2472 self.state.event(
2473 "setup",
2474 format!("worktree setup failed in {}: {e:#}", wt.display()),
2475 );
2476 self.state.save()?;
2477 return Err(e);
2478 }
2479 Ok(())
2480 }
2481
2482 async fn rollback_prep(&mut self, repo: &Path) {
2484 let root = self.state.worktree_root();
2485 for c in std::mem::take(&mut self.state.candidates) {
2486 if matches!(
2487 git::worktree_remove_clean(repo, &c.worktree).await,
2488 Ok(true)
2489 ) {
2490 git::branch_delete(repo, &c.branch).await.ok();
2491 } else {
2492 self.state.candidates.push(c);
2493 }
2494 }
2495 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496 for j in 1..=seats {
2497 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2498 .await
2499 .ok();
2500 }
2501 for k in 1..=self.state.config.graph.advisors {
2502 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2503 .await
2504 .ok();
2505 }
2506 }
2507
2508 async fn prep(&mut self) -> Result<()> {
2509 if let Some(reason) = &self.state.prep_failure {
2510 bail!(
2511 "candidate preparation previously failed: {reason}; correct the source before retrying"
2512 );
2513 }
2514 if self
2515 .state
2516 .prep_complete
2517 .unwrap_or(!self.state.candidates.is_empty())
2518 {
2519 return Ok(());
2520 }
2521 let repo = self.state.repo.clone();
2522 let _coordination = self.coordinate(&repo).await?;
2525 let base = self.state.base_commit.clone();
2526 let plan = self.state.sources.plan(&repo, &base).await?;
2527 self.rollback_prep(&repo).await;
2528 if !self.state.candidates.is_empty() {
2529 bail!(
2530 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531 );
2532 }
2533 self.state.prep_complete = Some(false);
2534 self.state.status = RunStatus::Prep;
2535 self.state.save()?;
2536 let repo = self.state.repo.clone();
2537 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538 let root = self.state.worktree_root();
2539 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541 let hooks_dir = self.state.dir().join("hooks");
2544 if self.state.config.blind.commit_msg_hook {
2545 std::fs::create_dir_all(&hooks_dir)
2546 .with_context(|| format!("create {}", hooks_dir.display()))?;
2547 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548 let path = hooks_dir.join("commit-msg");
2549 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550 make_executable(&path)?;
2551 git::acquire_worktree_config(&repo).await?;
2559 self.state.enabled_worktree_config = true;
2560 }
2561
2562 ensure_setup_config(&mut self.state, &repo).await?;
2565
2566 for (index, (spec, label)) in self
2567 .roles
2568 .implementers
2569 .clone()
2570 .into_iter()
2571 .zip(labels)
2572 .enumerate()
2573 {
2574 let branch = self.state.branch_for(label);
2575 let worktree = root.join(format!("cand-{label}"));
2576 self.state.event(
2580 "prep-intent",
2581 serde_json::json!({
2582 "branch": branch,
2583 "worktree": worktree,
2584 "start": start,
2585 "picks": plan.picks,
2586 })
2587 .to_string(),
2588 );
2589 self.state.save()?;
2590 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591 if self.state.config.blind.commit_msg_hook {
2592 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593 }
2594 git::local_exclude(&worktree, "/.magi/").await?;
2595 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596 git::worktree_remove_clean(&repo, &worktree).await.ok();
2599 git::branch_delete(&repo, &branch).await.ok();
2600 self.rollback_prep(&repo).await;
2601 self.state.status = RunStatus::Blocked;
2602 self.state
2603 .event("prep", format!("worktree setup failed: {e:#}"));
2604 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605 self.state.save()?;
2606 return Err(e);
2607 }
2608 self.state.candidates.push(Candidate {
2609 index,
2610 label,
2611 agent: spec.id.clone(),
2612 branch,
2613 worktree,
2614 summary: String::new(),
2615 stat: String::new(),
2616 files: 0,
2617 commits: 0,
2618 empty: false,
2619 failed: None,
2620 verified_noop: None,
2621 duration_ms: 0,
2622 folded: false,
2623 });
2624 self.state.save()?;
2625 for pick in &plan.picks {
2626 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627 {
2628 self.state.status = RunStatus::Blocked;
2629 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630 self.state
2631 .event("prep", format!("cannot apply referenced commit: {e}"));
2632 self.state.save()?;
2633 return Err(e);
2634 }
2635 }
2636 }
2637
2638 for j in 1..=self.roles.judges.len() {
2639 let wt = root.join(format!("judge-{j}"));
2640 if !wt.exists() {
2641 git::worktree_add_detached(&repo, &wt, &base).await?;
2642 self.setup_seat_worktree(&repo, &wt, true).await?;
2643 }
2644 }
2645
2646 if self.state.config.graph.advise {
2654 for k in 1..=self.state.config.graph.advisors {
2655 let wt = root.join(format!("advisor-{k}"));
2656 if !wt.exists() {
2657 git::worktree_add_detached(&repo, &wt, &base).await?;
2658 self.setup_seat_worktree(&repo, &wt, true).await?;
2659 }
2660 }
2661 }
2662
2663 let authors: Vec<&str> = self
2668 .roles
2669 .implementers
2670 .iter()
2671 .map(|a| a.id.as_str())
2672 .collect();
2673 let overlap: Vec<String> = self
2674 .roles
2675 .judges
2676 .iter()
2677 .enumerate()
2678 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680 .collect();
2681 if !overlap.is_empty() {
2682 let note = format!(
2683 "{} also authored a candidate; blind, but the panel is less \
2684 independent than {} distinct agents would be",
2685 overlap.join(", "),
2686 self.roles.judges.len()
2687 );
2688 self.state.event("prep", note);
2689 }
2690
2691 self.state.event(
2692 "prep",
2693 format!(
2694 "{} candidates, {} judges, base {} ({})",
2695 self.state.candidates.len(),
2696 self.roles.judges.len(),
2697 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698 self.state.base_branch
2699 ),
2700 );
2701 self.state.prep_complete = Some(true);
2702 self.state.status = RunStatus::Implementing;
2703 self.state.save()?;
2704 Ok(())
2705 }
2706
2707 async fn advise(&mut self) -> Result<()> {
2740 let implement_untouched = self
2741 .state
2742 .candidates
2743 .iter()
2744 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745 if !self.state.config.graph.advise || self.state.advise_attempted {
2746 return Ok(());
2747 }
2748 if !implement_untouched {
2749 self.state.event(
2750 "advise",
2751 "skipping the design-deliberation stage: at least one \
2752 candidate already shows implementation progress, so this \
2753 run is past the point the stage exists to run before"
2754 .to_owned(),
2755 );
2756 self.state.advise_attempted = true;
2757 self.state.save()?;
2758 return Ok(());
2759 }
2760 let run_id = self.state.id.clone();
2761 let prompts = self.state.config.prompts.clone();
2762 let instruction = self.state.instruction.clone();
2763 let language = self.state.config.graph.language.clone();
2764 let root = self.state.worktree_root();
2765 let n = self.state.config.graph.advisors;
2766 let where_recorded = self.state.dir().join("run.json");
2767
2768 let seats = match self.state.config.advisors() {
2769 Ok(seats) if !seats.is_empty() => seats,
2770 Ok(_) => {
2771 self.state.event(
2772 "advise",
2773 format!(
2774 "[graph] advisors is 0; skipping the design-deliberation \
2775 stage and continuing without a synthesis brief (see {})",
2776 where_recorded.display()
2777 ),
2778 );
2779 self.state.advise_attempted = true;
2780 self.state.save()?;
2781 return Ok(());
2782 }
2783 Err(e) => {
2784 self.state.event(
2785 "advise",
2786 format!(
2787 "could not resolve advisor seats ({e:#}); continuing \
2788 without a design-deliberation brief (see {})",
2789 where_recorded.display()
2790 ),
2791 );
2792 self.state.advise_attempted = true;
2793 self.state.save()?;
2794 return Ok(());
2795 }
2796 };
2797
2798 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799 let artifacts = agent::artifacts_dir(&self.state.dir());
2800 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802 let mut jobs = Vec::new();
2803 for (i, spec) in seats.iter().cloned().enumerate() {
2804 let seat_key = format!("advisor-{}", i + 1);
2805 let seat = self.seat(&seat_key, &spec.id);
2806 jobs.push(SeatJob {
2807 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808 spec,
2809 seat,
2810 cwd: worktrees[i % worktrees.len()].clone(),
2811 timeout,
2812 allow_write: false,
2813 sessions: false,
2814 artifacts: artifacts.clone(),
2815 stem: seat_key,
2816 handover: None,
2817 });
2818 }
2819
2820 self.state.event(
2821 "advise",
2822 format!(
2823 "{} advisor seat(s) sketching a design in parallel",
2824 jobs.len()
2825 ),
2826 );
2827 let mut quota_losses = Vec::new();
2828 let cache = self.state.config.cache_dir();
2829 let ctx = WaveCtx {
2830 carry_seats: false,
2831 pauses: Some((&self.pause, &self.interrupt)),
2832 run: &run_id,
2833 node: "advise",
2834 prompts: &prompts,
2835 cache: cache.as_deref(),
2836 round: None,
2837 };
2838 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839 let results = ask_json_wave::<Proposal>(
2840 jobs,
2841 Arc::clone(&self.sem),
2842 self.state.config.graph.retries,
2843 &advisor_roster,
2844 &ctx,
2845 &mut quota_losses,
2846 &mut self.state,
2847 &|p: &Proposal| p.validate(),
2848 )
2849 .await;
2850 if self.state.resource_block.is_some() {
2851 return self.stop_for_resource();
2852 }
2853 self.state.quota.extend(quota_losses);
2854
2855 let mut records = Vec::with_capacity(results.len());
2856 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857 let agent_id = seat.agent.clone();
2858 self.state.seats.insert(seat.key.clone(), seat);
2859 match res {
2860 Ok((proposal, out)) => {
2861 self.state
2862 .event("advise", format!("advisor-{} proposed a design", i + 1));
2863 records.push(advise::AdvisorRecord::proposed(
2864 i + 1,
2865 agent_id,
2866 proposal,
2867 out.duration_ms,
2868 ));
2869 }
2870 Err(e) => {
2871 self.state.event(
2872 "advise",
2873 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874 );
2875 records.push(advise::AdvisorRecord::failed(
2876 i + 1,
2877 agent_id,
2878 e.to_string(),
2879 ));
2880 }
2881 }
2882 }
2883
2884 let mut advice = advise::Advice {
2885 records,
2886 synthesis: None,
2887 };
2888 if advice.proposals().is_empty() {
2889 self.state.event(
2890 "advise",
2891 "no advisor produced a usable proposal; continuing without a \
2892 synthesis brief"
2893 .to_owned(),
2894 );
2895 } else {
2896 match self
2897 .synthesize_brief(
2898 &advice,
2899 &instruction,
2900 &language,
2901 &worktrees[0],
2902 &artifacts,
2903 &run_id,
2904 &prompts,
2905 cache.as_deref(),
2906 )
2907 .await
2908 {
2909 Ok(Some(text)) => {
2910 self.state.event(
2911 "advise",
2912 "synthesized a design brief for the implementer".to_owned(),
2913 );
2914 advice.synthesis = Some(text);
2915 }
2916 Ok(None) => {
2917 self.state.event(
2918 "advise",
2919 "the synthesis seat produced nothing usable; continuing \
2920 without a design brief"
2921 .to_owned(),
2922 );
2923 }
2924 Err(e) => {
2925 if e.is::<crate::repo_lock::ResourceBlocked>() {
2926 return Err(e);
2927 }
2928 self.state.event(
2929 "advise",
2930 format!("could not synthesize a design brief: {e:#}"),
2931 );
2932 }
2933 }
2934 }
2935 advise::apply_reflection(&mut advice);
2936
2937 self.state.advice = Some(advice);
2938 self.state.advise_attempted = true;
2939 self.state.save()?;
2940 Ok(())
2941 }
2942
2943 #[allow(clippy::too_many_arguments)]
2955 async fn synthesize_brief(
2956 &mut self,
2957 advice: &advise::Advice,
2958 instruction: &str,
2959 language: &str,
2960 cwd: &Path,
2961 artifacts: &Path,
2962 run_id: &str,
2963 prompts: &Prompts,
2964 cache: Option<&Path>,
2965 ) -> Result<Option<String>> {
2966 let chain = agent::pick_chain(
2967 &self.state.config.agents,
2968 self.state.config.roles.synthesizer.as_ref(),
2969 &agent::installed,
2970 "synthesizer",
2971 )?;
2972 let proposals = advice.proposals();
2973 let mut prompt = prompt::with_overlay(
2974 prompt::synthesize_brief(instruction, &proposals, language),
2975 prompts.overlay("advise"),
2976 );
2977 if cache.is_some() {
2978 prompt.push('\n');
2983 prompt.push_str(&prompt::build_cache_note("advise", false));
2984 }
2985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986 let mut last = None;
2991 for (n, spec) in chain.iter().enumerate() {
2992 if n > 0 {
2993 self.state
2994 .event("advise", format!("synthesis falling back to {}", spec.id));
2995 }
2996 let mut seat = self.seat("advise-synthesis", &spec.id);
2997 let wait = self.repository_wait();
2998 let outcome = wait
2999 .scope(agent::invoke(
3000 spec,
3001 &mut seat,
3002 &Invocation {
3003 coordination: self.state.config.coordination.as_ref(),
3004 fetched_base: None,
3005 cwd,
3006 prompt: &prompt,
3007 timeout,
3008 allow_write: false,
3009 sessions: false,
3010 artifacts,
3011 stem: &if n == 0 {
3012 "advise-synthesis".to_owned()
3013 } else {
3014 format!("advise-synthesis-{}", spec.id)
3015 },
3016 run: run_id,
3017 node: "advise",
3018 cache_dir: None,
3019 attachments: &[],
3020 writable: &[],
3021 },
3022 ))
3023 .await;
3024 if outcome.is_ok() {
3025 self.state.seats.insert(seat.key.clone(), seat);
3026 }
3027 let advance = agent::chain_advances(&outcome);
3028 last = Some(outcome);
3029 if !advance {
3030 break;
3031 }
3032 }
3033 let out = last.expect("a chain holds at least one agent")?;
3036 if !out.usable() {
3037 return Ok(None);
3038 }
3039 let text =
3040 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041 Ok((!text.trim().is_empty()).then_some(text))
3042 }
3043
3044 async fn implement(&mut self) -> Result<()> {
3047 self.persist_candidates().await?;
3048 let run_id = self.state.id.clone();
3053 let prompts = self.state.config.prompts.clone();
3054 let todo: Vec<usize> = self
3055 .state
3056 .candidates
3057 .iter()
3058 .enumerate()
3059 .filter(|(_, c)| {
3060 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061 && c.failed.is_none()
3062 && !c.empty
3063 })
3064 .map(|(i, _)| i)
3065 .collect();
3066 if todo.is_empty() {
3067 return self.after_implement();
3068 }
3069 self.state.status = RunStatus::Implementing;
3070
3071 let language = self.state.config.graph.language.clone();
3072 let github_language = self.state.config.graph.github_language.clone();
3073 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074 let sessions = self.state.config.graph.sessions;
3075 let artifacts = agent::artifacts_dir(&self.state.dir());
3076 let brief = self
3080 .state
3081 .advice
3082 .as_ref()
3083 .and_then(|a| a.synthesis.as_deref())
3084 .map(str::to_owned);
3085 let attachments = self.state.attachments.clone();
3086
3087 let mut jobs = Vec::new();
3088 for &i in &todo {
3089 let (index, label, worktree) = {
3090 let c = &self.state.candidates[i];
3091 (c.index, c.label, c.worktree.clone())
3092 };
3093 let spec = self.roles.implementers[index].clone();
3094 let seat_key = format!("impl-{label}");
3095 let seat = self.seat(&seat_key, &spec.id);
3096 let instruction = seeded_instruction(&self.state);
3097 jobs.push(SeatJob {
3098 spec,
3099 seat,
3100 prompt: prompt::implement_in(
3101 &instruction,
3102 &worktree.to_string_lossy(),
3103 &language,
3104 &github_language,
3105 brief.as_deref(),
3106 &attachments,
3107 ),
3108 cwd: worktree,
3109 timeout,
3110 allow_write: true,
3111 sessions,
3112 artifacts: artifacts.clone(),
3113 stem: format!("impl-{label}"),
3114 handover: None,
3115 });
3116 }
3117
3118 self.state.event(
3119 "implement",
3120 format!("{} candidates in parallel", jobs.len()),
3121 );
3122 let mut sent = jobs.clone();
3128 let cache = self.state.config.cache_dir();
3129 let ctx = WaveCtx {
3130 carry_seats: false,
3131 pauses: Some((&self.pause, &self.interrupt)),
3132 run: &run_id,
3133 node: "implement",
3134 prompts: &prompts,
3135 cache: cache.as_deref(),
3136 round: None,
3137 };
3138 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139 if !wave_blocked(&results) {
3140 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141 .await;
3142 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143 .await?;
3144 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145 .await;
3146 }
3147 if !results.is_empty()
3150 && results
3151 .iter()
3152 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153 {
3154 let losses: Vec<_> = results
3155 .iter()
3156 .map(|(_, seat, out)| {
3157 let AgentOutcome::Quota(o) = out else {
3158 unreachable!()
3159 };
3160 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161 })
3162 .collect();
3163 self.state.quota.extend(losses.clone());
3164 for (_, seat, _) in results {
3165 self.state.seats.insert(seat.key.clone(), seat);
3166 }
3167 for &i in &todo {
3168 let label = self.state.candidates[i].label;
3169 self.state.interrupted_implementations.insert(label);
3170 self.state
3171 .candidate_preservation
3172 .insert(label, "machine outage awaiting persistence".into());
3173 }
3174 block_machine_outage(&mut self.state, "implement", &losses);
3177 self.state.save()?;
3178 self.persist_candidates().await?;
3179 for &i in &todo {
3180 self.state
3181 .interrupted_implementations
3182 .insert(self.state.candidates[i].label);
3183 }
3184 return self.stop_for_resource();
3185 }
3186 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189 && results
3190 .iter()
3191 .any(|(_, _, out)| FailClass::of(out).is_some());
3192 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194 if self
3198 .state
3199 .candidate_preservation
3200 .contains_key(&self.state.candidates[i].label)
3201 {
3202 self.state.seats.insert(seat.key.clone(), seat);
3203 }
3204 continue;
3205 }
3206 let seat_key = seat.key.clone();
3207 let agent = seat.agent.clone();
3216 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217 self.state.seats.insert(seat.key.clone(), seat);
3218 let label = self.state.candidates[i].label;
3219 self.state.interrupted_implementations.remove(&label);
3220
3221 let (summary, duration, failed, verified_claim) = match out {
3222 AgentOutcome::Ok(o) => {
3223 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224 let failed = (!o.usable()).then(|| {
3225 if o.timed_out {
3226 "agent timed out".to_owned()
3227 } else {
3228 format!(
3229 "agent exited with {}",
3230 crate::notices::exit_status(o.exit_code)
3231 )
3232 }
3233 });
3234 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235 (text, o.duration_ms, failed, verified_claim)
3236 }
3237 AgentOutcome::Dropped(o) => {
3243 let why = o
3244 .dropped
3245 .as_ref()
3246 .map(|d| d.why.as_str())
3247 .unwrap_or("the CLI ended the stream without delivering its answer");
3248 (
3249 String::new(),
3250 o.duration_ms,
3251 Some(format!("the CLI dropped the stream ({why})")),
3252 None,
3253 )
3254 }
3255 AgentOutcome::Quota(o) => {
3256 self.state.quota.push(QuotaLoss::from_report(
3257 seat_key,
3258 "implement",
3259 o.quota.as_ref(),
3260 ));
3261 (
3262 String::new(),
3263 o.duration_ms,
3264 Some("rate limited (quota); produced no change".to_owned()),
3265 None,
3266 )
3267 }
3268 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269 (String::new(), 0, Some(e), None)
3270 }
3271 };
3272
3273 let c = &mut self.state.candidates[i];
3276 if !exhausted_the_fallback_chain {
3277 c.agent = agent;
3278 }
3279 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280 c.duration_ms = duration;
3281 c.failed = failed;
3282 c.verified_noop = verified_claim;
3283 self.state
3284 .candidate_preservation
3285 .insert(label, "awaiting persistence".to_owned());
3286 }
3287 self.state.save()?;
3288 self.persist_candidates().await?;
3289
3290 if resource_blocked {
3291 return self.stop_for_resource();
3292 }
3293 if selected_failed {
3294 self.state.status = RunStatus::Blocked;
3295 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296 self.state.save()?;
3297 bail!(
3298 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299 );
3300 }
3301 self.after_implement()
3302 }
3303
3304 async fn persist_candidates(&mut self) -> Result<()> {
3306 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307 for label in labels {
3308 let i = self
3309 .state
3310 .candidates
3311 .iter()
3312 .position(|c| c.label == label)
3313 .context("pending candidate is missing")?;
3314 let c = self.state.candidates[i].clone();
3315 let interrupted = self.state.interrupted_implementations.contains(&label);
3316 let result = async {
3317 let wait = crate::repo_lock::WaitPolicy::new(
3318 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319 || false,
3320 );
3321 let _coordination =
3322 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323 if let Err(e) = git::snapshot_uncommitted(
3326 &c.worktree,
3327 &self
3328 .state
3329 .dir()
3330 .join("artifacts")
3331 .join(format!("candidate-{label}-recovery")),
3332 )
3333 .await
3334 {
3335 self.state.event(
3336 "preserve",
3337 format!("candidate {label} snapshot incomplete: {e:#}"),
3338 );
3339 }
3340 let rescued = git::rescue_commit(
3341 &c.worktree,
3342 &format!("magi: candidate {label} (uncommitted work)"),
3343 )
3344 .await?;
3345 self.state.note_withheld("implement", &rescued.withheld);
3346 let base = &self.state.base_commit;
3347 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352 if c.empty && commits > 0 && !patch.trim().is_empty()
3353 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355 || self.state.tally.is_some())
3356 {
3357 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361 }
3362 let c = &mut self.state.candidates[i];
3363 c.commits = commits;
3364 c.stat = stat;
3365 c.files = files;
3366 let no_changes = commits == 0 || patch.trim().is_empty();
3367 c.empty = no_changes && !interrupted;
3368 if interrupted && no_changes {
3369 c.failed = None;
3370 c.verified_noop = None;
3371 }
3372 if !c.empty {
3373 c.failed = None;
3374 c.verified_noop = None;
3375 }
3376 self.state.event(
3377 "preserve",
3378 format!("candidate {label}: {files} files, {commits} commits"),
3379 );
3380 Ok::<_, anyhow::Error>(())
3381 }
3382 .await;
3383 match result {
3384 Ok(()) => {
3385 self.state.candidate_preservation.remove(&label);
3386 if !interrupted || self.state.candidates[i].files > 0 {
3387 self.state.interrupted_implementations.remove(&label);
3388 }
3389 }
3390 Err(e) => {
3391 let reason = format!("candidate {label} preservation failed: {e:#}");
3392 self.state.event("preserve", &reason);
3393 self.state.candidate_preservation.insert(label, reason);
3394 }
3395 }
3396 if self.state.candidate_preservation.is_empty()
3397 && self.state.status == RunStatus::Blocked
3398 && self.state.candidates.iter().all(|c| c.failed.is_none())
3399 {
3400 self.state.status = RunStatus::Implementing;
3404 self.state.event(
3405 "preserve",
3406 "candidate persistence recovered; implementation can continue",
3407 );
3408 }
3409 self.state.save()?;
3410 }
3411 if !self.state.candidate_preservation.is_empty() {
3412 self.state.status = RunStatus::Blocked;
3413 self.state.save()?;
3414 bail!(
3415 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416 );
3417 }
3418 Ok(())
3419 }
3420
3421 async fn resume_undelivered(
3449 &mut self,
3450 results: &mut [(usize, SeatState, AgentOutcome)],
3451 sent: &[SeatJob],
3452 prompts: &Prompts,
3453 run_id: &str,
3454 ) {
3455 for (wi, seat, out) in results.iter_mut() {
3456 let Some(dropped) = (match &*out {
3457 AgentOutcome::Dropped(o) => o.dropped.clone(),
3458 _ => None,
3459 }) else {
3460 continue;
3461 };
3462 let Some(job) = sent.get(*wi) else { continue };
3463 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465 self.state.event(
3466 "implement",
3467 format!(
3468 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469 work is in the tree",
3470 seat.key, dropped.output_tokens, dropped.why
3471 ),
3472 );
3473 continue;
3474 }
3475 if !has_context(&job.spec, seat, job.sessions) {
3483 self.state.event(
3484 "implement",
3485 format!(
3486 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487 is no session left to resume",
3488 seat.key, dropped.output_tokens, dropped.why
3489 ),
3490 );
3491 continue;
3492 }
3493 self.state.event(
3494 "implement",
3495 format!(
3496 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497 conversation",
3498 seat.key, dropped.output_tokens, dropped.why
3499 ),
3500 );
3501 let mut retry = job.clone();
3502 retry.seat = seat.clone();
3503 retry.prompt = prompt::resume_after_drop(&dropped.why);
3504 retry.timeout = retry_budget(job.timeout, true);
3505 retry.stem = format!("{}-resume", job.stem);
3506 let cache = self.state.config.cache_dir();
3507 let ctx = WaveCtx {
3508 carry_seats: false,
3509 pauses: Some((&self.pause, &self.interrupt)),
3510 run: run_id,
3511 node: "implement",
3512 prompts,
3513 cache: cache.as_deref(),
3514 round: None,
3515 };
3516 let (resumed_seat, resumed) =
3517 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518 *seat = resumed_seat;
3519 *out = resumed;
3520 }
3521 }
3522
3523 async fn resume_seat_handovers(
3588 &mut self,
3589 results: &mut [(usize, SeatState, AgentOutcome)],
3590 sent: &mut [SeatJob],
3591 prompts: &Prompts,
3592 run_id: &str,
3593 ) -> Result<()> {
3594 let instruction = seeded_instruction(&self.state);
3595 let language = self.state.config.graph.language.clone();
3596 let github_language = self.state.config.graph.github_language.clone();
3597 let brief = self
3598 .state
3599 .advice
3600 .as_ref()
3601 .and_then(|a| a.synthesis.as_deref())
3602 .map(str::to_owned);
3603 let attachments = self.state.attachments.clone();
3604 for (wi, seat, out) in results.iter_mut() {
3605 let others: BTreeSet<String> = sent
3608 .iter()
3609 .enumerate()
3610 .filter(|(j, _)| j != wi)
3611 .map(|(_, j)| j.spec.id.clone())
3612 .collect();
3613 let Some(job) = sent.get_mut(*wi) else {
3614 continue;
3615 };
3616 let start = self
3621 .roles
3622 .implementer_roster
3623 .iter()
3624 .position(|s| s.id == job.spec.id)
3625 .unwrap_or(0);
3626 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627 let mut fallback_attempt = 0usize;
3628 let mut prev: Option<FailClass> = None;
3629 while let Some(cur) = FailClass::of(&*out) {
3630 if !should_hand_over(prev.as_ref(), &cur) {
3631 break;
3632 }
3633 let Some(next) =
3634 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635 .cloned()
3636 else {
3637 break;
3638 };
3639 tried.insert(next.id.clone());
3640 fallback_attempt += 1;
3641
3642 let coordination = self.coordinate(&job.cwd).await?;
3643 if let Ok(r) = git::rescue_commit(
3644 &job.cwd,
3645 &format!(
3646 "magi: candidate {} (uncommitted work before {} fallback)",
3647 seat.key,
3648 if cur == FailClass::Quota {
3649 "quota"
3650 } else {
3651 "handover"
3652 }
3653 ),
3654 )
3655 .await
3656 {
3657 self.state.note_withheld("implement", &r.withheld);
3658 }
3659 drop(coordination);
3660
3661 record_handover(
3662 &mut self.state,
3663 "implement",
3664 &seat.key,
3665 &seat.agent,
3666 &next.id,
3667 &cur,
3668 &fail_reason(&*out),
3669 );
3670 prev = Some(cur.clone());
3671
3672 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674 job.spec = next.clone();
3682 let mut retry = job.clone();
3683 retry.seat = new_seat;
3684 retry.prompt = prompt::implement_in(
3685 &instruction,
3686 &job.cwd.to_string_lossy(),
3687 &language,
3688 &github_language,
3689 brief.as_deref(),
3690 &attachments,
3691 );
3692 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693 let cache = self.state.config.cache_dir();
3694 let ctx = WaveCtx {
3695 carry_seats: false,
3696 pauses: Some((&self.pause, &self.interrupt)),
3697 run: run_id,
3698 node: "implement",
3699 prompts,
3700 cache: cache.as_deref(),
3701 round: None,
3702 };
3703 let (fallback_seat, fallback_out) = run_one(
3704 retry,
3705 Arc::clone(&self.sem),
3706 &ctx,
3707 &mut self.state,
3708 fallback_attempt,
3709 )
3710 .await;
3711 *seat = fallback_seat;
3712 *out = fallback_out;
3713 }
3714 }
3715 Ok(())
3716 }
3717
3718 async fn resume_unconfirmed_commands(
3742 &mut self,
3743 results: &mut [(usize, SeatState, AgentOutcome)],
3744 sent: &[SeatJob],
3745 prompts: &Prompts,
3746 run_id: &str,
3747 ) {
3748 for (wi, seat, out) in results.iter_mut() {
3749 let AgentOutcome::Ok(o) = &*out else {
3750 continue;
3751 };
3752 if !has_unconfirmed_command(&o.commands) {
3753 continue;
3754 }
3755 let Some(job) = sent.get(*wi) else { continue };
3756 if !has_context(&job.spec, seat, job.sessions) {
3757 self.state.event(
3758 "implement",
3759 format!(
3760 "{}: the reply named a command whose own CLI never confirmed the exit \
3761 status of, but there is no session left to resume",
3762 seat.key
3763 ),
3764 );
3765 continue;
3766 }
3767 self.state.event(
3768 "implement",
3769 format!(
3770 "{}: the reply named a command whose own CLI never confirmed the exit \
3771 status of; resuming the conversation",
3772 seat.key
3773 ),
3774 );
3775 let mut retry = job.clone();
3776 retry.seat = seat.clone();
3777 retry.prompt = prompt::resume_incomplete(
3778 "a command in your last reply had no confirmed exit status",
3779 );
3780 retry.timeout = retry_budget(job.timeout, true);
3781 retry.stem = format!("{}-confirm", job.stem);
3782 let cache = self.state.config.cache_dir();
3783 let ctx = WaveCtx {
3784 carry_seats: false,
3785 pauses: Some((&self.pause, &self.interrupt)),
3786 run: run_id,
3787 node: "implement",
3788 prompts,
3789 cache: cache.as_deref(),
3790 round: None,
3791 };
3792 let (resumed_seat, resumed) =
3793 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794 *seat = resumed_seat;
3795 *out = resumed;
3796 }
3797 }
3798
3799 async fn continue_fix_report(
3820 &mut self,
3821 mut seat: SeatState,
3822 parse_err: String,
3823 job: &SeatJob,
3824 prompts: &Prompts,
3825 run_id: &str,
3826 round: usize,
3827 ) -> (
3828 SeatState,
3829 Option<FixReport>,
3830 Option<String>,
3831 ContinuationRecord,
3832 ) {
3833 let mut last_err = parse_err;
3834 let mut cumulative_wait_ms = 0u64;
3835 let mut attempts = 0usize;
3836 loop {
3837 if !has_context(&job.spec, &seat, job.sessions) {
3838 self.state.event(
3839 "fix",
3840 format!(
3841 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842 session left to resume into"
3843 ),
3844 );
3845 let outcome = if attempts == 0 {
3846 ContinuationOutcome::NoSession
3847 } else {
3848 ContinuationOutcome::Exhausted
3849 };
3850 return (
3851 seat,
3852 None,
3853 Some(format!("unparsable fix report: {last_err}")),
3854 ContinuationRecord {
3855 attempts,
3856 cumulative_wait_ms,
3857 outcome,
3858 },
3859 );
3860 }
3861 if attempts >= MAX_FIX_CONTINUATIONS {
3862 self.state.event(
3863 "fix",
3864 format!(
3865 "round {round}: fixer's reply still had no adoption report after \
3866 {attempts} continuation(s) ({last_err}); giving up"
3867 ),
3868 );
3869 return (
3870 seat,
3871 None,
3872 Some(format!(
3873 "unparsable fix report after {attempts} continuation(s): {last_err}"
3874 )),
3875 ContinuationRecord {
3876 attempts,
3877 cumulative_wait_ms,
3878 outcome: ContinuationOutcome::Exhausted,
3879 },
3880 );
3881 }
3882 attempts += 1;
3883 self.state.event(
3884 "fix",
3885 format!(
3886 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888 ),
3889 );
3890 let mut retry = job.clone();
3891 retry.seat = seat.clone();
3892 retry.prompt = prompt::resume_incomplete(&last_err);
3893 retry.timeout = retry_budget(job.timeout, true);
3894 retry.stem = format!("{}-continue{attempts}", job.stem);
3895 let cache = self.state.config.cache_dir();
3896 let ctx = WaveCtx {
3897 carry_seats: false,
3898 pauses: Some((&self.pause, &self.interrupt)),
3899 run: run_id,
3900 node: "fix",
3901 prompts,
3902 cache: cache.as_deref(),
3903 round: Some(round),
3904 };
3905 let (resumed_seat, resumed_out) = run_one(
3906 retry,
3907 Arc::clone(&self.sem),
3908 &ctx,
3909 &mut self.state,
3910 attempts,
3911 )
3912 .await;
3913 seat = resumed_seat;
3914 match resumed_out {
3915 AgentOutcome::Ok(o) => {
3916 cumulative_wait_ms += o.duration_ms;
3917 match verdict::extract_json::<FixReport>(&o.text) {
3918 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919 self.state.event(
3920 "fix",
3921 format!(
3922 "round {round}: fixer's adoption report recovered after \
3923 {attempts} continuation(s)"
3924 ),
3925 );
3926 return (
3927 seat,
3928 Some(report),
3929 None,
3930 ContinuationRecord {
3931 attempts,
3932 cumulative_wait_ms,
3933 outcome: ContinuationOutcome::Resumed,
3934 },
3935 );
3936 }
3937 Ok(_) => {
3945 last_err = "the reply parsed, but it reported a command whose own CLI \
3946 never confirmed an exit status"
3947 .to_owned();
3948 }
3949 Err(e) => last_err = e.to_string(),
3950 }
3951 }
3952 AgentOutcome::Quota(o) => {
3953 cumulative_wait_ms += o.duration_ms;
3954 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956 self.state.quota.push(loss);
3957 self.state.event(
3958 "fix",
3959 format!(
3960 "round {round}: continuation rate limited (quota); not retrying now"
3961 ),
3962 );
3963 return (
3964 seat,
3965 None,
3966 Some("rate limited (quota) while recovering the fix report".to_owned()),
3967 ContinuationRecord {
3968 attempts,
3969 cumulative_wait_ms,
3970 outcome: ContinuationOutcome::QuotaLost,
3971 },
3972 );
3973 }
3974 AgentOutcome::Dropped(o) => {
3975 cumulative_wait_ms += o.duration_ms;
3976 let why = o
3977 .dropped
3978 .as_ref()
3979 .map(|d| d.why.as_str())
3980 .unwrap_or("the CLI ended the stream without delivering its answer");
3981 last_err = format!("the CLI dropped the stream ({why})");
3982 }
3983 AgentOutcome::Failed(e) => last_err = e,
3984 AgentOutcome::ResourceBlocked(e) => {
3987 return (
3988 seat,
3989 None,
3990 Some(format!("the continuation was not started: {e}")),
3991 ContinuationRecord {
3992 attempts: attempts.saturating_sub(1),
3993 cumulative_wait_ms,
3994 outcome: ContinuationOutcome::NoSession,
3995 },
3996 );
3997 }
3998 }
3999 }
4000 }
4001
4002 fn after_implement(&mut self) -> Result<()> {
4003 if !self.state.candidate_preservation.is_empty() {
4004 bail!("candidate preservation incomplete; judging is unsafe");
4005 }
4006 if self.state.leaks.is_empty() {
4008 let cfg = self.state.config.blind.clone();
4009 let mut leaks = Vec::new();
4010 for c in &self.state.candidates {
4011 let Some(patch) =
4012 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013 else {
4014 continue;
4015 };
4016 leaks.extend(blind::scan(
4017 &format!("candidate {} patch", c.label),
4018 &patch,
4019 &cfg.vendor_tokens,
4020 ));
4021 }
4022 if !leaks.is_empty() {
4023 let summary = leaks
4024 .iter()
4025 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026 .collect::<Vec<_>>()
4027 .join(", ");
4028 match cfg.on_leak {
4029 LeakPolicy::Fail => {
4030 self.state.status = RunStatus::Failed;
4031 self.state
4032 .event("blind", format!("vendor text in a patch: {summary}"));
4033 self.state.leaks = leaks;
4034 self.state.save()?;
4035 self.settle_questions();
4036 bail!(
4037 "blind.on_leak = \"fail\" and vendor text reached a \
4038 judged patch: {summary}"
4039 );
4040 }
4041 LeakPolicy::Redact => self.state.event(
4042 "blind",
4043 format!("redacting vendor text for judging: {summary}"),
4044 ),
4045 LeakPolicy::Warn => self.state.event(
4046 "blind",
4047 format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048 ),
4049 }
4050 self.state.leaks = leaks;
4051 }
4052 }
4053
4054 if self.state.viable().is_empty() {
4055 if self.state.all_candidates_verified_noop() {
4056 self.state.status = RunStatus::VerifiedNoop;
4067 self.state.save()?;
4068 self.settle_questions();
4069 return Ok(());
4070 }
4071 self.state.status = RunStatus::Failed;
4072 self.state.save()?;
4073 self.settle_questions();
4074 bail!("no candidate produced a change; nothing to judge");
4075 }
4076 self.state.status = RunStatus::Judging;
4077 self.state.save()?;
4078 Ok(())
4079 }
4080
4081 async fn judge(&mut self) -> Result<()> {
4084 let run_id = self.state.id.clone();
4089 let prompts = self.state.config.prompts.clone();
4090 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091 return Ok(());
4092 }
4093 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094 if viable.len() == 1 {
4095 self.state.judge_skipped = true;
4102 self.state.event(
4103 "judge",
4104 format!(
4105 "only candidate {} produced a change; judging skipped",
4106 viable[0].label
4107 ),
4108 );
4109 self.state.save()?;
4110 return Ok(());
4111 }
4112 self.state.status = RunStatus::Judging;
4113
4114 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115 let language = self.state.config.graph.language.clone();
4116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117 let sessions = self.state.config.graph.sessions;
4118 let artifacts = agent::artifacts_dir(&self.state.dir());
4119 let root = self.state.worktree_root();
4120 let base_short = short(&self.state.base_commit);
4121
4122 let mut jobs = Vec::new();
4123 let mut orders = Vec::new();
4124 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128 let seat_key = format!("judge-{}", j + 1);
4129 let seat = self.seat(&seat_key, &spec.id);
4130 jobs.push(SeatJob {
4131 prompt: prompt::judge(
4132 &self.state.instruction,
4133 &views,
4134 self.roles.judges.len(),
4135 &base_short,
4136 &language,
4137 ),
4138 spec,
4139 seat,
4140 cwd: root.join(format!("judge-{}", j + 1)),
4141 timeout,
4142 allow_write: false,
4143 sessions,
4144 artifacts: artifacts.clone(),
4145 stem: format!("judge-{}", j + 1),
4146 handover: None,
4147 });
4148 }
4149
4150 self.state.event(
4151 "judge",
4152 format!(
4153 "{} judges ranking {} candidates blind",
4154 jobs.len(),
4155 viable.len()
4156 ),
4157 );
4158 let labels_for_check = labels.clone();
4159 let mut quota_losses = Vec::new();
4160 let cache = self.state.config.cache_dir();
4161 let ctx = WaveCtx {
4162 carry_seats: false,
4163 pauses: Some((&self.pause, &self.interrupt)),
4164 run: &run_id,
4165 node: "judge",
4166 prompts: &prompts,
4167 cache: cache.as_deref(),
4168 round: None,
4169 };
4170 let results = ask_json_wave::<Ranking>(
4171 jobs,
4172 Arc::clone(&self.sem),
4173 self.state.config.graph.retries,
4174 &self.roles.judge_roster,
4175 &ctx,
4176 &mut quota_losses,
4177 &mut self.state,
4178 &move |r: &Ranking| r.validate(&labels_for_check),
4179 )
4180 .await;
4181 if self.state.resource_block.is_some() {
4182 return self.stop_for_resource();
4183 }
4184 self.state.quota.extend(quota_losses);
4185
4186 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187 let agent_id = seat.agent.clone();
4188 self.state.seats.insert(seat.key.clone(), seat);
4189 let mut record = Judgement {
4190 judge: j + 1,
4191 seat: format!("judge-{}", j + 1),
4192 agent: agent_id,
4193 ranking: Vec::new(),
4194 reasons: BTreeMap::new(),
4195 confidence: None,
4196 strengths: BTreeMap::new(),
4197 order: orders[j].clone(),
4198 failed: None,
4199 duration_ms: 0,
4200 };
4201 match res {
4202 Ok((ranking, out)) => {
4203 record.ranking = ranking.normalized();
4204 record.reasons = ranking.reasons;
4205 record.strengths = ranking.strengths;
4206 record.confidence = ranking.confidence;
4207 record.duration_ms = out.duration_ms;
4208 self.state.event(
4209 "judge",
4210 format!(
4211 "judge {} ranked {}",
4212 j + 1,
4213 record.ranking.iter().collect::<String>()
4214 ),
4215 );
4216 }
4217 Err(e) => {
4218 record.failed = Some(e.to_string());
4219 self.state
4220 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221 }
4222 }
4223 self.state.judgements.push(record);
4224 self.state.save()?;
4225 }
4226 Ok(())
4227 }
4228
4229 async fn deliberate(&mut self) -> Result<()> {
4232 if self.hold_base_sync().await? {
4233 return Ok(());
4234 }
4235 let run_id = self.state.id.clone();
4240 let prompts = self.state.config.prompts.clone();
4241 let rounds = self.state.config.graph.deliberate_rounds;
4242 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244 return Ok(());
4245 }
4246 let tops: Vec<char> = self
4247 .state
4248 .judgements
4249 .iter()
4250 .filter_map(|j| j.ranking.first().copied())
4251 .collect();
4252 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254 self.state.event(
4255 "deliberate",
4256 format!("judges agreed on {} outright; no deliberation", tops[0]),
4257 );
4258 }
4259 self.state.status = RunStatus::Voting;
4260 self.state.save()?;
4261 return Ok(());
4262 }
4263
4264 self.state.status = RunStatus::Deliberating;
4265 self.state.event(
4266 "deliberate",
4267 format!(
4268 "split: first choices were {} — opening {rounds} round(s)",
4269 tops.iter().collect::<String>()
4270 ),
4271 );
4272
4273 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274 let language = self.state.config.graph.language.clone();
4275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276 let sessions = self.state.config.graph.sessions;
4277 let artifacts = agent::artifacts_dir(&self.state.dir());
4278 let root = self.state.worktree_root();
4279 let base_short = short(&self.state.base_commit);
4280
4281 for round in self.state.deliberation.len() + 1..=rounds {
4285 let mut turns: Vec<DeliberationTurn> = Vec::new();
4286 let mut round_losses = Vec::new();
4287 let mut asked = 0;
4288 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289 if self.state.judgements[j].failed.is_some() {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let mut seat = self.seat(&seat_key, &spec.id);
4295 let transcript = self.transcript(&turns, j);
4296 let build = |context: Option<&str>| {
4297 prompt::deliberate(
4298 &self.state.instruction,
4299 context,
4300 &transcript,
4301 round,
4302 rounds,
4303 &language,
4304 )
4305 };
4306 let block = self.candidate_block(&viable, &base_short);
4307 let full = build(Some(&block));
4308 let text = if has_context(&spec, &seat, sessions) {
4309 build(None)
4310 } else {
4311 full.clone()
4312 };
4313 let job = SeatJob {
4314 spec,
4315 seat: seat.clone(),
4316 prompt: text,
4317 cwd: root.join(format!("judge-{}", j + 1)),
4318 timeout,
4319 allow_write: false,
4320 sessions,
4321 artifacts: artifacts.clone(),
4322 stem: format!("delib-{round}-judge-{}", j + 1),
4323 handover: Some(full),
4324 };
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "deliberate",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let mut losses = Vec::new();
4339 asked += 1;
4340 let mut results = ask_wave_with::<String>(
4341 vec![job],
4342 Arc::clone(&self.sem),
4343 0,
4344 &self.roles.judge_roster,
4345 &ctx,
4346 &mut losses,
4347 &mut self.state,
4348 &|text: &str| {
4349 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 return self.stop_for_resource();
4355 }
4356 round_losses.extend(losses.iter().cloned());
4357 self.state.quota.extend(losses);
4358 let (updated, res, _) = results.pop().expect("one job in, one result out");
4359 seat = updated;
4360 let agent_id = seat.agent.clone();
4361 self.state.seats.insert(seat.key.clone(), seat);
4362 let body = match res {
4363 Ok((body, _)) => body,
4364 Err(e) => {
4367 self.state
4368 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369 continue;
4370 }
4371 };
4372 let tentative = verdict::extract_json::<Position>(&body)
4373 .ok()
4374 .and_then(|p| p.tentative)
4375 .and_then(|s| s.trim().chars().next())
4376 .map(|c| c.to_ascii_uppercase());
4377 self.state.event(
4378 "deliberate",
4379 format!(
4380 "round {round}: judge {} now favours {}",
4381 j + 1,
4382 tentative.map_or("—".to_owned(), |c| c.to_string())
4383 ),
4384 );
4385 turns.push(DeliberationTurn {
4386 judge: j + 1,
4387 agent: agent_id,
4388 body: blind::sanitize_prose(&body, &self.state.config.blind),
4389 tentative,
4390 });
4391 }
4392 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394 return self.stop_for_resource();
4395 }
4396 self.state
4397 .deliberation
4398 .push(DeliberationRound { round, turns });
4399 self.state.save()?;
4400 }
4401
4402 self.state.status = RunStatus::Voting;
4403 self.state.save()?;
4404 Ok(())
4405 }
4406
4407 async fn vote(&mut self) -> Result<()> {
4410 if self.hold_base_sync().await? {
4411 return Ok(());
4412 }
4413 let run_id = self.state.id.clone();
4418 let prompts = self.state.config.prompts.clone();
4419 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420 return Ok(());
4421 }
4422 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423 if viable.len() == 1 {
4424 return Ok(());
4425 }
4426 self.state.status = RunStatus::Voting;
4427
4428 let language = self.state.config.graph.language.clone();
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let root = self.state.worktree_root();
4433 let base_short = short(&self.state.base_commit);
4434 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436 let mut jobs = Vec::new();
4437 let mut seats_at = Vec::new();
4438 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439 if self
4440 .state
4441 .judgements
4442 .get(j)
4443 .is_some_and(|r| r.failed.is_some())
4444 {
4445 continue;
4446 }
4447 let seat_key = format!("judge-{}", j + 1);
4448 let spec = self.occupant(&seat_key, spec);
4449 let seat = self.seat(&seat_key, &spec.id);
4450 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451 let text = if has_context(&spec, &seat, sessions) {
4452 prompt::final_vote(&viable, &language)
4453 } else {
4454 full.clone()
4455 };
4456 jobs.push(SeatJob {
4457 spec,
4458 seat,
4459 prompt: text,
4460 cwd: root.join(format!("judge-{}", j + 1)),
4461 timeout,
4462 allow_write: false,
4463 sessions,
4464 artifacts: artifacts.clone(),
4465 stem: format!("vote-judge-{}", j + 1),
4466 handover: Some(full),
4467 });
4468 seats_at.push(j);
4469 }
4470
4471 self.state.event(
4472 "vote",
4473 format!(
4474 "collecting {} final votes one by one, privately",
4475 jobs.len()
4476 ),
4477 );
4478 let allowed = viable.clone();
4479 let mut quota_losses = Vec::new();
4480 let cache = self.state.config.cache_dir();
4481 let ctx = WaveCtx {
4482 carry_seats: false,
4483 pauses: Some((&self.pause, &self.interrupt)),
4484 run: &run_id,
4485 node: "vote",
4486 prompts: &prompts,
4487 cache: cache.as_deref(),
4488 round: None,
4489 };
4490 let results = ask_json_wave::<FinalVote>(
4491 jobs,
4492 Arc::clone(&self.sem),
4493 self.state.config.graph.retries,
4494 &self.roles.judge_roster,
4495 &ctx,
4496 &mut quota_losses,
4497 &mut self.state,
4498 &move |v: &FinalVote| match v.label() {
4499 Some(c) if allowed.contains(&c) => Ok(()),
4500 other => bail!("vote {other:?} is not one of {allowed:?}"),
4501 },
4502 )
4503 .await;
4504 if self.state.resource_block.is_some() {
4505 return self.stop_for_resource();
4506 }
4507 self.state.quota.extend(quota_losses);
4508
4509 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510 let agent_id = seat.agent.clone();
4511 self.state.seats.insert(seat.key.clone(), seat);
4512 let initial = self
4513 .state
4514 .judgements
4515 .get(j)
4516 .and_then(|r| r.ranking.first().copied());
4517 let mut record = VoteRecord {
4518 judge: j + 1,
4519 agent: agent_id,
4520 vote: None,
4521 reason: String::new(),
4522 changed: false,
4523 };
4524 match res {
4525 Ok((v, _)) => {
4526 record.vote = v.label();
4527 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529 self.state.event(
4530 "vote",
4531 format!(
4532 "judge {} voted {}{}",
4533 j + 1,
4534 record.vote.unwrap_or('?'),
4535 if record.changed { " (changed)" } else { "" }
4536 ),
4537 );
4538 }
4539 Err(e) => {
4540 self.state
4541 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542 }
4543 }
4544 self.state.votes.push(record);
4545 self.state.save()?;
4546 }
4547 Ok(())
4548 }
4549
4550 fn tally(&mut self) -> Result<()> {
4553 if self.state.tally.is_some() {
4554 return Ok(());
4555 }
4556 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557 let tops: Vec<char> = self
4558 .state
4559 .judgements
4560 .iter()
4561 .filter_map(|j| j.ranking.first().copied())
4562 .collect();
4563 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568 let mut cast: Vec<char> = Vec::new();
4569 for (i, j) in self.state.judgements.iter().enumerate() {
4570 let vote = self
4571 .state
4572 .votes
4573 .iter()
4574 .find(|v| v.judge == i + 1)
4575 .and_then(|v| v.vote)
4576 .or_else(|| j.ranking.first().copied());
4577 if let Some(v) = vote {
4578 *first_choice.entry(v).or_insert(0) += 1;
4579 cast.push(v);
4580 }
4581 }
4582
4583 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584 for j in &self.state.judgements {
4585 let n = j.ranking.len();
4586 for (pos, label) in j.ranking.iter().enumerate() {
4587 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588 }
4589 }
4590
4591 let best = first_choice.values().copied().max().unwrap_or(0);
4592 let mut leaders: Vec<char> = first_choice
4593 .iter()
4594 .filter(|(_, v)| **v == best)
4595 .map(|(k, _)| *k)
4596 .collect();
4597 let mut tie_break = None;
4598 if leaders.len() > 1 {
4599 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600 let borda_leaders: Vec<char> = leaders
4601 .iter()
4602 .copied()
4603 .filter(|l| borda[l] == top_borda)
4604 .collect();
4605 tie_break = Some(if borda_leaders.len() == 1 {
4606 format!(
4607 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608 leaders.len()
4609 )
4610 } else {
4611 format!(
4612 "{} way tie on both first-choice votes and Borda points, broken by label order",
4613 leaders.len()
4614 )
4615 });
4616 leaders = borda_leaders;
4617 leaders.sort_unstable();
4618 }
4619 let winner = *leaders
4620 .first()
4621 .or(viable.first())
4622 .context("no candidate to declare a winner from")?;
4623
4624 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626 let deliberated = !self.state.deliberation.is_empty();
4627
4628 let quota_seats: std::collections::BTreeSet<&str> =
4632 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633 let mut present = 0usize;
4634 for (i, j) in self.state.judgements.iter().enumerate() {
4635 if quota_seats.contains(j.seat.as_str()) {
4636 continue;
4637 }
4638 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639 let voted = self
4640 .state
4641 .votes
4642 .iter()
4643 .any(|v| v.judge == i + 1 && v.vote.is_some());
4644 if ranked || voted {
4645 present += 1;
4646 }
4647 }
4648 let needs_quorum = viable.len() > 1;
4654 let judges_total = if needs_quorum {
4655 self.roles.judges.len()
4656 } else {
4657 0
4658 };
4659 let quorum = if needs_quorum {
4660 judges_total / 2 + 1
4661 } else {
4662 0
4663 };
4664 let met_quorum = !needs_quorum || present >= quorum;
4665 let uncontested = (!needs_quorum).then(|| {
4666 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667 });
4668
4669 self.state.event(
4670 "tally",
4671 match &uncontested {
4672 Some(reason) => format!("winner {winner} — {reason}"),
4673 None => format!(
4674 "winner {winner} — votes {} | initial {} | {} changed | \
4675 {present}/{judges_total} judges{}",
4676 first_choice
4677 .iter()
4678 .map(|(k, v)| format!("{k}:{v}"))
4679 .collect::<Vec<_>>()
4680 .join(" "),
4681 if unanimous_initial {
4682 "unanimous"
4683 } else {
4684 "split"
4685 },
4686 changed_votes,
4687 if met_quorum {
4688 String::new()
4689 } else {
4690 format!(" — below quorum ({quorum} required)")
4691 },
4692 ),
4693 },
4694 );
4695 if !met_quorum {
4696 self.state.event(
4697 "stall",
4698 format!(
4699 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700 the run stops here, resumable"
4701 ),
4702 );
4703 }
4704 self.state.tally = Some(Tally {
4705 first_choice,
4706 borda,
4707 winner,
4708 rankings: tops.len(),
4709 unanimous_initial,
4710 deliberated,
4711 changed_votes,
4712 unanimous_final,
4713 tie_break,
4714 judges: judges_total,
4715 present,
4716 quorum,
4717 met_quorum,
4718 uncontested,
4719 });
4720 self.state.status = if met_quorum {
4721 RunStatus::Reviewing
4722 } else {
4723 RunStatus::Stalled
4724 };
4725 self.state.save()?;
4726 Ok(())
4727 }
4728
4729 #[allow(clippy::too_many_lines)]
4750 async fn recover_stall(&mut self) -> Result<bool> {
4751 let run_id = self.state.id.clone();
4756 let prompts = self.state.config.prompts.clone();
4757 let quota_seats: BTreeSet<&str> =
4762 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763 let absent: Vec<String> = self
4764 .state
4765 .judgements
4766 .iter()
4767 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768 .map(|j| j.seat.clone())
4769 .collect();
4770 if absent.is_empty() {
4771 return Ok(false);
4772 }
4773 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774 if viable.len() <= 1 {
4775 return Ok(false);
4776 }
4777 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778 let language = self.state.config.graph.language.clone();
4779 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780 let sessions = self.state.config.graph.sessions;
4781 let artifacts = agent::artifacts_dir(&self.state.dir());
4782 let root = self.state.worktree_root();
4783 let base_short = short(&self.state.base_commit);
4784 let candidates: Vec<Candidate> = viable.clone();
4785
4786 let mut positions: Vec<usize> = absent
4788 .iter()
4789 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790 .collect();
4791 if positions.is_empty() {
4792 return Ok(false);
4793 }
4794 positions.sort_unstable();
4795 positions.dedup();
4796
4797 let mut judge_jobs = Vec::new();
4799 for &j in &positions {
4800 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802 let seat_key = format!("judge-{}", j + 1);
4803 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804 let seat = self.seat(&seat_key, &spec.id);
4805 judge_jobs.push(SeatJob {
4806 spec,
4807 seat,
4808 prompt: prompt::judge(
4809 &self.state.instruction,
4810 &views,
4811 self.roles.judges.len(),
4812 &base_short,
4813 &language,
4814 ),
4815 cwd: root.join(seat_key),
4816 timeout,
4817 allow_write: false,
4818 sessions,
4819 artifacts: artifacts.clone(),
4820 stem: format!("judge-{}-recover", j + 1),
4821 handover: None,
4822 });
4823 }
4824
4825 let labels_for_check = labels.clone();
4826 let mut judge_losses = Vec::new();
4827 let retries = self.state.config.graph.retries;
4828 let cache = self.state.config.cache_dir();
4829 let ctx = WaveCtx {
4830 carry_seats: false,
4831 pauses: Some((&self.pause, &self.interrupt)),
4832 run: &run_id,
4833 node: "judge",
4834 prompts: &prompts,
4835 cache: cache.as_deref(),
4836 round: None,
4837 };
4838 let results = ask_json_wave::<Ranking>(
4839 judge_jobs,
4840 Arc::clone(&self.sem),
4841 retries,
4842 &self.roles.judge_roster,
4843 &ctx,
4844 &mut judge_losses,
4845 &mut self.state,
4846 &move |r: &Ranking| r.validate(&labels_for_check),
4847 )
4848 .await;
4849 if self.state.resource_block.is_some() {
4850 self.stop_for_resource()?;
4851 return Ok(false);
4852 }
4853
4854 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857 let agent_id = seat.agent.clone();
4858 self.state.seats.insert(seat.key.clone(), seat);
4859 let record = &mut self.state.judgements[j];
4860 match res {
4861 Ok((ranking, out)) => {
4862 record.agent = agent_id;
4863 record.ranking = ranking.normalized();
4864 record.reasons = ranking.reasons;
4865 record.strengths = ranking.strengths;
4866 record.confidence = ranking.confidence;
4867 record.failed = None;
4868 record.duration_ms = out.duration_ms;
4869 recovered.insert(j);
4870 self.state.event(
4871 "recover",
4872 format!("judge {} ranked again after the limit", j + 1),
4873 );
4874 }
4875 Err(e) => {
4876 self.state
4877 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878 }
4879 }
4880 }
4881
4882 let mut vote_jobs = Vec::new();
4884 let mut vote_pos: Vec<usize> = Vec::new();
4885 for &j in &recovered {
4886 let seat_key = format!("judge-{}", j + 1);
4887 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888 let seat = self.seat(&seat_key, &spec.id);
4889 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890 let text = if has_context(&spec, &seat, sessions) {
4891 prompt::final_vote(&labels, &language)
4892 } else {
4893 full.clone()
4894 };
4895 vote_jobs.push(SeatJob {
4896 spec,
4897 seat,
4898 prompt: text,
4899 cwd: root.join(seat_key),
4900 timeout,
4901 allow_write: false,
4902 sessions,
4903 artifacts: artifacts.clone(),
4904 stem: format!("vote-judge-{}-recover", j + 1),
4905 handover: Some(full),
4906 });
4907 vote_pos.push(j);
4908 }
4909 let allowed = labels.clone();
4910 let mut vote_losses = Vec::new();
4911 let vote_retries = self.state.config.graph.retries;
4912 let vote_cache = self.state.config.cache_dir();
4913 let ctx = WaveCtx {
4914 carry_seats: false,
4915 pauses: Some((&self.pause, &self.interrupt)),
4916 run: &run_id,
4917 node: "vote",
4918 prompts: &prompts,
4919 cache: vote_cache.as_deref(),
4920 round: None,
4921 };
4922 let votes = ask_json_wave::<FinalVote>(
4923 vote_jobs,
4924 Arc::clone(&self.sem),
4925 vote_retries,
4926 &self.roles.judge_roster,
4927 &ctx,
4928 &mut vote_losses,
4929 &mut self.state,
4930 &move |v: &FinalVote| match v.label() {
4931 Some(c) if allowed.contains(&c) => Ok(()),
4932 other => bail!("vote {other:?} is not one of {allowed:?}"),
4933 },
4934 )
4935 .await;
4936 if self.state.resource_block.is_some() {
4937 self.stop_for_resource()?;
4938 return Ok(false);
4939 }
4940 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941 let agent_id = seat.agent.clone();
4942 self.state.seats.insert(seat.key.clone(), seat);
4943 match res {
4944 Ok((v, _)) => {
4945 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946 rec.vote = v.label();
4947 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948 } else {
4949 self.state.votes.push(VoteRecord {
4950 judge: j + 1,
4951 agent: agent_id,
4952 vote: v.label(),
4953 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954 changed: false,
4955 });
4956 }
4957 self.state.event(
4958 "recover",
4959 format!("judge {} voted again after the limit", j + 1),
4960 );
4961 }
4962 Err(e) => {
4963 self.state
4964 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965 }
4966 }
4967 }
4968
4969 let recovered_keys: BTreeSet<String> = recovered
4973 .iter()
4974 .map(|&j| format!("judge-{}", j + 1))
4975 .collect();
4976 self.state
4977 .quota
4978 .retain(|q| !recovered_keys.contains(&q.seat));
4979 for loss in judge_losses.into_iter().chain(vote_losses) {
4983 if recovered_keys.contains(&loss.seat) {
4984 continue;
4985 }
4986 self.state.quota.retain(|q| q.seat != loss.seat);
4987 self.state.quota.push(loss);
4988 }
4989
4990 self.state.tally = None;
4992 self.tally()?;
4993 Ok(self
4994 .state
4995 .tally
4996 .as_ref()
4997 .map(|t| t.met_quorum)
4998 .unwrap_or(false))
4999 }
5000
5001 async fn graft(&mut self) -> Result<()> {
5017 let Err(e) = self.graft_inner().await else {
5018 return Ok(());
5019 };
5020 if e.is::<crate::repo_lock::ResourceBlocked>() {
5021 return Err(e);
5022 }
5023 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024 self.state.status = RunStatus::Blocked;
5025 self.state.event("graft", format!("blocked: {e:#}"));
5026 self.state.save()?;
5027 return Err(e);
5028 }
5029 if let Some(rec) = self.state.graft.clone()
5034 && !rec.outcome.settled()
5035 {
5036 if let Some(winner) = self.state.winner().cloned() {
5037 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038 if re.is::<crate::repo_lock::ResourceBlocked>() {
5039 return Err(re);
5040 }
5041 self.state.status = RunStatus::Blocked;
5042 self.state.event("graft", format!("blocked: {re:#}"));
5043 self.state.save()?;
5044 return Err(re);
5045 }
5046 }
5047 if let Some(g) = self.state.graft.as_mut() {
5048 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049 }
5050 }
5051 self.state.event("graft", format!("skipped: {e:#}"));
5052 if let Err(e) = self.state.save() {
5053 tracing::warn!("save after graft: {e:#}");
5054 }
5055 Ok(())
5056 }
5057
5058 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061 let _coordination = self.coordinate(wt).await?;
5062 crate::recovery::restore_repair(wt, pre)
5063 .await
5064 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065 }
5066
5067 fn graft_skipped(&mut self, why: &str) {
5069 let message = format!("skipped: {why}");
5070 if !self
5071 .state
5072 .events
5073 .iter()
5074 .any(|e| e.node == "graft" && e.message == message)
5075 {
5076 self.state.event("graft", message);
5077 }
5078 }
5079
5080 async fn graft_inner(&mut self) -> Result<()> {
5081 if let Some(rec) = self.state.graft.clone()
5084 && !rec.outcome.settled()
5085 {
5086 if self.state.reviews.is_empty()
5090 && let Some(winner) = self.state.winner().cloned()
5091 {
5092 self.graft_restore(&winner.worktree, &rec.pre).await?;
5093 }
5094 if rec.outcome == GraftOutcome::WaitingForRestore {
5095 self.state.graft = None;
5096 self.state.save()?;
5097 } else {
5098 if let Some(g) = self.state.graft.as_mut() {
5099 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100 }
5101 self.state.event(
5102 "graft",
5103 "rolled back: the previous round was interrupted; the winner is untouched",
5104 );
5105 self.state.save()?;
5106 return Ok(());
5107 }
5108 }
5109 let plan = match graft::plan(&self.state) {
5110 Ok(plan) => plan,
5111 Err(skip) => {
5112 if skip != graft::Skip::AlreadyDone {
5113 self.graft_skipped(skip.reason());
5114 self.state.save()?;
5115 }
5116 return Ok(());
5117 }
5118 };
5119 let Some(winner) = self.state.winner().cloned() else {
5120 return Ok(());
5121 };
5122 let wt = winner.worktree.clone();
5123 if !git::is_clean(&wt).await? {
5124 self.graft_skipped("the winner's worktree has uncommitted changes");
5125 self.state.save()?;
5126 return Ok(());
5127 }
5128 let pre = git::rev_parse(&wt, "HEAD").await?;
5129 self.state.graft = Some(GraftRecord {
5132 pre: pre.clone(),
5133 post: None,
5134 lines: None,
5135 limit: None,
5136 agent: None,
5137 outcome: GraftOutcome::Running,
5138 requested: graft::requested(&plan),
5139 adoption: None,
5140 adoption_error: None,
5141 });
5142 self.state.event(
5143 "graft",
5144 format!(
5145 "porting strengths of candidate(s) {} onto {}",
5146 plan.strengths
5147 .iter()
5148 .map(|(l, _, _)| l.to_string())
5149 .collect::<Vec<_>>()
5150 .join(", "),
5151 plan.winner
5152 ),
5153 );
5154 self.state.save()?;
5155
5156 let fix_prompt = prompt::graft(
5157 &self.state.instruction,
5158 plan.winner,
5159 &plan.strengths,
5160 &self.state.config.graph.language,
5161 );
5162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163 let sessions = self.state.config.graph.sessions;
5164 let artifacts = agent::artifacts_dir(&self.state.dir());
5165 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166 let (_, seat, out) = self
5167 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168 SeatJob {
5169 prompt: fix_prompt.clone(),
5170 spec,
5171 seat,
5172 cwd: wt.clone(),
5173 timeout,
5174 allow_write: true,
5175 sessions,
5176 artifacts: artifacts.clone(),
5177 stem: "graft".to_owned(),
5178 handover: None,
5179 }
5180 })
5181 .await;
5182 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183 self.state.graft.as_mut().expect("graft record").outcome =
5186 GraftOutcome::WaitingForRestore;
5187 return self.stop_for_resource();
5188 }
5189 let mut why: Option<String> = None;
5190 match &out {
5191 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192 why = Some("the fixer produced nothing usable".to_owned());
5193 }
5194 AgentOutcome::Ok(_) => {}
5195 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196 AgentOutcome::Quota(o) => {
5197 self.state.quota.push(QuotaLoss::from_report(
5198 seat.key.clone(),
5199 "graft",
5200 o.quota.as_ref(),
5201 ));
5202 why = Some("rate limited (quota)".to_owned());
5203 }
5204 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205 why = Some(format!(
5206 "the fixer failed: {}",
5207 e.lines()
5208 .next()
5209 .unwrap_or("")
5210 .chars()
5211 .take(160)
5212 .collect::<String>()
5213 ));
5214 }
5215 }
5216 let agent_id = seat.agent.clone();
5217 self.state.seats.insert(seat.key.clone(), seat);
5218
5219 let mut post = None;
5220 let mut lines = None;
5221 let _coordination = self.coordinate(&wt).await?;
5222 if why.is_none() {
5223 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224 if let Ok(r) = &rescued {
5225 self.state.note_withheld("graft", &r.withheld);
5226 }
5227 let head = git::rev_parse(&wt, "HEAD").await?;
5228 let dirty = match &rescued {
5232 Ok(r) => git::status_porcelain(&wt)
5233 .await
5234 .map(|st| {
5235 st.lines().any(|l| {
5236 !r.withheld
5237 .iter()
5238 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239 })
5240 })
5241 .unwrap_or(true),
5242 Err(_) => true,
5243 };
5244 if dirty {
5245 why = Some("the fixer left changes that could not be committed".to_owned());
5246 } else if head == pre {
5247 why = Some("the fixer changed nothing".to_owned());
5248 } else {
5249 lines = git::changed_lines(&wt, &pre, &head).await;
5250 post = Some(head);
5251 }
5252 }
5253 let (adoption, adoption_error) = match &out {
5254 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255 _ => (None, None),
5256 };
5257 let requested = graft::requested(&plan);
5258 let outcome = match why {
5259 None => {
5260 self.state.event(
5261 "graft",
5262 format!(
5263 "applied: {}",
5264 lines.map_or_else(
5265 || "changed lines unavailable".to_owned(),
5266 |n| format!("{n} changed lines")
5267 )
5268 ),
5269 );
5270 GraftOutcome::Applied
5271 }
5272 Some(why) => {
5273 crate::recovery::restore_repair(&wt, &pre)
5276 .await
5277 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278 self.state.event(
5279 "graft",
5280 format!("rolled back: {why}; the winner is untouched"),
5281 );
5282 GraftOutcome::RolledBack(why)
5283 }
5284 };
5285 self.state.graft = Some(GraftRecord {
5286 pre,
5287 post,
5288 lines,
5289 limit: None,
5290 agent: Some(agent_id),
5291 outcome,
5292 requested,
5293 adoption,
5294 adoption_error,
5295 });
5296 self.state.save()?;
5297 Ok(())
5298 }
5299
5300 async fn fold_losers(&mut self) -> Result<()> {
5303 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304 return Ok(());
5305 };
5306 if !self.state.candidate_preservation.is_empty() {
5307 bail!("candidate preservation incomplete; refusing cleanup");
5308 }
5309 let repo = self.state.repo.clone();
5310 let _coordination = self.coordinate(&repo).await?;
5311 let mut folded = Vec::new();
5312 for i in 0..self.state.candidates.len() {
5313 let c = &self.state.candidates[i];
5314 if c.label == winner || c.folded {
5315 continue;
5316 }
5317 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318 remove_preserved_worktree(&repo, &wt).await?;
5319 if git::branch_exists(&repo, &branch).await?
5320 && !git::branch_delete(&repo, &branch).await?
5321 {
5322 bail!("candidate branch removal refused");
5323 }
5324 self.state.candidates[i].folded = true;
5325 self.state.save()?;
5326 folded.push(label.to_string());
5327 }
5328 let root = self.state.worktree_root();
5330 for j in 1..=self.roles.judges.len() {
5331 let wt = root.join(format!("judge-{j}"));
5332 if wt.exists() {
5333 remove_preserved_worktree(&repo, &wt).await?;
5334 }
5335 }
5336 if self.state.config.graph.advise {
5339 for k in 1..=self.state.config.graph.advisors {
5340 let wt = root.join(format!("advisor-{k}"));
5341 if wt.exists() {
5342 remove_preserved_worktree(&repo, &wt).await?;
5343 }
5344 }
5345 }
5346 if !folded.is_empty() {
5347 self.state
5348 .event("fold", format!("folded candidates {}", folded.join(", ")));
5349 self.state.save()?;
5350 }
5351 Ok(())
5352 }
5353
5354 async fn hold_base_sync(&mut self) -> Result<bool> {
5357 self.hold_base_sync_as(None).await
5358 }
5359
5360 async fn hold_base_sync_kind(
5361 &mut self,
5362 kind: crate::base_sync_recovery::StopKind,
5363 ) -> Result<()> {
5364 self.hold_base_sync_as(Some(kind)).await?;
5365 Ok(())
5366 }
5367
5368 async fn hold_base_sync_as(
5369 &mut self,
5370 kind: Option<crate::base_sync_recovery::StopKind>,
5371 ) -> Result<bool> {
5372 if !self.state.base_sync_blocked() {
5373 self.state.base_sync_recovery.stop = None;
5374 return Ok(false);
5375 }
5376 let head = match self.state.winner() {
5377 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5378 .await
5379 .unwrap_or_default(),
5380 None => String::new(),
5381 };
5382 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5383 && !authorization.consumed
5384 && authorization.stop.head != head
5385 {
5386 authorization.refuse(format!(
5387 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5388 authorization.stop.head
5389 ));
5390 }
5391 self.state.record_base_sync_stop(&head);
5392 if let Some(kind) = kind {
5393 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5394 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5395 stop.kind = kind;
5396 }
5397 }
5398 self.state.save()?;
5399 Ok(true)
5400 }
5401
5402 async fn sync_to_base(&mut self) -> Result<()> {
5444 if self.state.status == RunStatus::AlreadyInBase {
5445 return Ok(());
5446 }
5447 self.hold_base_sync().await?;
5448 let conflicted = self
5449 .state
5450 .base_sync
5451 .as_ref()
5452 .is_some_and(|s| s.conflict.is_some());
5453 let Some(winner) = self.state.winner().cloned() else {
5454 return Ok(());
5455 };
5456
5457 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5458
5459 let repo = self.state.repo.clone();
5460 let remote = self.state.config.merge.remote.clone();
5461 let base_branch = self.state.base_branch.clone();
5462 let tracking = format!("{remote}/{base_branch}");
5463
5464 let fetched_base =
5465 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5466 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5471 return Ok(());
5472 };
5473
5474 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5475 self.state.ensure_base_episode();
5476 let mut pending = self
5477 .state
5478 .base_sync_recovery
5479 .operations
5480 .last()
5481 .filter(|o| !o.finished)
5482 .cloned();
5483 let scratch = self.state.dir().join("base-sync");
5484 if git::rebase_in_progress(&scratch).await {
5485 if pending.is_none()
5488 && self.state.base_sync_recovery.operations.is_empty()
5489 && !conflicted
5490 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5491 .await
5492 .ok()
5493 .as_deref()
5494 == Some(head.as_str())
5495 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5496 && git::is_clean(&winner.worktree).await?
5497 {
5498 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5499 if self.state.base_sync.is_none() {
5500 self.state.base_sync = Some(BaseSync {
5501 tip: saved_tip.clone(),
5502 behind: 0,
5503 attempts: 0,
5504 conflict: None,
5505 already_in: None,
5506 });
5507 }
5508 self.state.reserve_base_operation(&head, &saved_tip);
5509 self.state.event(
5510 "base-recovery",
5511 "imported standing legacy rebase as a spent synchronization reservation",
5512 );
5513 self.state.save()?;
5514 pending = self.state.base_sync_recovery.operations.last().cloned();
5515 }
5516 let expected = pending.as_ref().map(|op| op.tip.as_str());
5517 if expected.is_none()
5518 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5519 {
5520 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5521 if self.state.base_sync.is_none() {
5522 self.state.base_sync = Some(BaseSync {
5523 tip: tip.clone(),
5524 behind: 0,
5525 attempts: 0,
5526 conflict: None,
5527 already_in: None,
5528 });
5529 }
5530 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5531 self.state.event("land", why);
5532 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5533 .await?;
5534 return Ok(());
5535 }
5536 }
5537 if let Some(op) = &pending {
5538 let mut replay_proven =
5540 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5541 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5542 let wait = repository_wait(&self.state, None);
5543 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5544 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5545 Ok(true) => {
5546 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5547 self.state.save()?;
5548 git::worktree_remove(&repo, &scratch).await.ok();
5549 replay_proven = true;
5550 }
5551 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5552 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5553 }
5554 }
5555 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5556 if self.state.foreign_checkout.is_some() {
5557 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5558 .await
5559 .ok()
5560 .as_deref()
5561 == Some(op.head.as_str())
5562 && git::rev_parse(&repo, &format!("{head}^2"))
5563 .await
5564 .ok()
5565 .as_deref()
5566 == Some(op.tip.as_str());
5567 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5568 .await
5569 .is_ok_and(|m| m.is_empty())
5570 && let Ok(added) =
5571 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5572 {
5573 replay_proven = true;
5574 for commit in added {
5575 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5576 replay_proven = false;
5577 break;
5578 }
5579 }
5580 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5581 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5582 }
5583 }
5584 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5588 || (head != op.head
5589 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5590 .await?
5591 .is_empty())
5592 || (!git::is_clean(&winner.worktree).await?
5593 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5594 .await
5595 .is_ok_and(|o| o.ok()))
5596 || !replay_proven
5597 {
5598 let why = format!(
5599 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5600 op.id
5601 );
5602 let sync = self
5603 .state
5604 .base_sync
5605 .as_mut()
5606 .context("reserved synchronization has no lifetime accounting")?;
5607 sync.conflict = Some(why.clone());
5608 self.state.event("land", why);
5609 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5610 .await?;
5611 return Ok(());
5612 }
5613 if head != op.head {
5614 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5615 }
5616 tip = op.tip.clone();
5617 }
5618 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5619 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5620 if pending.is_none()
5621 && !conflicted
5622 && fetched_base
5623 && self
5624 .state
5625 .base_sync_recovery
5626 .episodes
5627 .last()
5628 .is_some_and(|e| e.boundary.is_some())
5629 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5630 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5631 && owner.execution_runs().last() == Some(&self.state.id)
5632 && owner.status != crate::queue::TaskStatus::Done
5633 && !owner.fresh_start
5634 && owner.review_branch.is_none()
5635 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5636 && let Some(approval) = owner.resumption.as_ref()
5637 && approval.instruction == owner.instruction
5638 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5639 && git::is_clean(&winner.worktree).await?
5640 && self.state.renew_base_episode(&task, approval, &head, &tip)
5641 {
5642 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5643 self.state.save()?;
5644 }
5645
5646 if (behind > 0 || conflicted || head == tip)
5655 && self
5656 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5657 .await?
5658 {
5659 return Ok(());
5660 }
5661 if conflicted && self.state.base_recovery_pending() {
5662 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5663 let owner =
5664 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5665 let owner_valid = owner.as_ref().is_ok_and(|t| {
5666 t.execution_runs().last() == Some(&self.state.id)
5667 && t.status != crate::queue::TaskStatus::Done
5668 && !t.fresh_start
5669 && t.review_branch.is_none()
5670 });
5671 let invalidation = if !owner_valid {
5672 Some("latest task ownership changed")
5673 } else if !owner.as_ref().is_ok_and(|t| {
5674 authorization
5675 .generation
5676 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5677 }) {
5678 Some("owner approval generation changed")
5679 } else if authorization.question.as_ref().is_some_and(|question| {
5680 !crate::base_recovery_control::decision_current(
5681 &crate::queue::Queue::at(crate::run::home().join("queue")),
5682 &self.state,
5683 &authorization.task,
5684 question,
5685 )
5686 }) {
5687 Some("recovery decision ownership, routing or stop evidence changed")
5688 } else if authorization.request.is_some()
5689 && !owner
5690 .as_ref()
5691 .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5692 {
5693 Some("owner decision changed")
5694 } else if let Some(scope) = &authorization.scope {
5695 if scope.tip != tip {
5696 Some("fetched base changed")
5697 } else if !self
5698 .state
5699 .base_sync_recovery
5700 .episodes
5701 .last()
5702 .is_some_and(|e| e.id == scope.episode)
5703 {
5704 Some("active synchronization interval changed")
5705 } else if !owner
5706 .as_ref()
5707 .is_ok_and(|t| t.instruction == scope.instruction)
5708 {
5709 Some("task instruction changed")
5710 } else {
5711 None
5712 }
5713 } else {
5714 None
5715 };
5716 if let Some(reason) = invalidation {
5717 let authorization = self
5718 .state
5719 .base_sync_recovery
5720 .authorizations
5721 .last_mut()
5722 .unwrap();
5723 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5724 self.state.save()?;
5725 return Ok(());
5726 }
5727 if !fetched_base
5728 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5729 || !git::is_clean(&winner.worktree).await?
5730 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5731 {
5732 return Ok(());
5733 }
5734 }
5735 let recovery = conflicted && self.state.consume_base_recovery(&head);
5736 if conflicted {
5737 self.state.save()?;
5739 if !recovery {
5740 return Ok(());
5741 }
5742 }
5743
5744 if behind == 0 {
5745 if let Some(from) = self
5754 .state
5755 .rebase_fixes
5756 .iter()
5757 .rev()
5758 .find_map(|r| r.from.clone())
5759 && from != head
5760 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5761 .await
5762 .is_ok_and(|o| o.ok())
5763 {
5764 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5765 }
5766 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5772 if let Some(why) = &conflict {
5773 self.state.status = RunStatus::Blocked;
5774 self.state.event("land", why.clone());
5775 }
5776 self.state.base_sync = Some(BaseSync {
5777 tip,
5778 behind: 0,
5779 attempts,
5780 conflict,
5781 already_in: None,
5782 });
5783 if pending.is_some() {
5784 self.state.finish_base_operation(Some(&head));
5785 }
5786 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5787 .await?;
5788 self.state.save()?;
5789 return Ok(());
5790 }
5791
5792 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5793 let why = format!(
5794 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5795 synchronization(s); synchronizing again would only race it",
5796 winner.branch
5797 );
5798 self.state.status = RunStatus::Blocked;
5799 self.state.base_sync = Some(BaseSync {
5800 tip,
5801 behind,
5802 attempts,
5803 conflict: Some(why.clone()),
5804 already_in: None,
5805 });
5806 self.state.event("land", why);
5807 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5808 .await?;
5809 self.state.save()?;
5810 return Ok(());
5811 }
5812
5813 if self.state.foreign_checkout.is_some() {
5814 let wait = repository_wait(&self.state, None);
5817 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5818 if self.state.base_sync.is_none() {
5819 self.state.base_sync = Some(BaseSync {
5820 tip: tip.clone(),
5821 behind,
5822 attempts: 0,
5823 conflict: None,
5824 already_in: None,
5825 });
5826 }
5827 self.state.reserve_base_operation(&head, &tip);
5828 self.state.save()?;
5829 let merged = git::git_raw(
5830 &winner.worktree,
5831 &[
5832 "-c",
5833 "core.hooksPath=/dev/null",
5834 "merge",
5835 "--no-ff",
5836 "--no-autostash",
5837 "--no-edit",
5838 "-m",
5839 &format!("Merge {tracking} before verifying"),
5840 &tip,
5841 ],
5842 )
5843 .await?;
5844 let conflict = if merged.ok() {
5845 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5846 self.state.event(
5847 "land",
5848 format!("merged {tracking} into {} before verifying", winner.branch),
5849 );
5850 None
5851 } else {
5852 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5853 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5854 }
5855 let why = format!(
5856 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5857 winner.branch,
5858 merged.stderr.chars().take(600).collect::<String>()
5859 );
5860 self.state.status = RunStatus::Blocked;
5861 self.state.event("land", &why);
5862 Some(why)
5863 };
5864 let synced = conflict.is_none();
5865 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5866 self.state
5867 .finish_base_operation(synced.then_some(fresh.as_str()));
5868 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5869 self.state.base_sync = Some(BaseSync {
5870 tip,
5871 behind: if synced { 0 } else { behind },
5872 attempts,
5873 conflict,
5874 already_in: None,
5875 });
5876 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5877 .await?;
5878 self.state.save()?;
5879 return Ok(());
5880 }
5881
5882 self.state.event(
5883 "land",
5884 format!(
5885 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5886 winner.branch
5887 ),
5888 );
5889 self.state.save()?;
5890
5891 let branch_tracking = format!("{remote}/{}", winner.branch);
5897 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5898 let remote_tip = if self.state.task_pr_branch().is_none()
5899 && matches!(&fetched_branch, Ok(o) if o.ok())
5900 {
5901 git::rev_parse(&repo, &branch_tracking).await.ok()
5902 } else {
5903 None
5904 };
5905 if let Some(theirs) = &remote_tip
5909 && !git::is_ancestor(&repo, theirs, &head).await
5910 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5911 .await
5912 .is_ok_and(|missing| missing.is_empty())
5913 {
5914 let why = format!(
5915 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5916 them",
5917 short(theirs),
5918 winner.branch
5919 );
5920 self.state.status = RunStatus::Blocked;
5921 self.state.base_sync = Some(BaseSync {
5922 tip,
5923 behind,
5924 attempts,
5925 conflict: Some(why.clone()),
5926 already_in: None,
5927 });
5928 self.state.event("land", why);
5929 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5930 .await?;
5931 self.state.save()?;
5932 return Ok(());
5933 }
5934
5935 if self.state.base_sync.is_none() {
5936 self.state.base_sync = Some(BaseSync {
5937 tip: tip.clone(),
5938 behind,
5939 attempts: 0,
5940 conflict: None,
5941 already_in: None,
5942 });
5943 }
5944 self.state.reserve_base_operation(&head, &tip);
5945 self.state.save()?;
5946 let scratch = self.state.dir().join("base-sync");
5947 let rebased =
5948 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5949 .await
5950 {
5951 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5952 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5953 Err(e) => Err(e),
5954 };
5955 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5956 let stop_kind = match &rebased {
5957 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5958 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5959 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5960 };
5961 match rebased {
5962 Ok(None) => {
5963 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5967 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5968 self.state.finish_base_operation(Some(&fresh));
5969 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5970 let mut conflict = None;
5971 if let Some(pinned) = &remote_tip {
5972 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5973 match pushed {
5974 Ok(o) if o.ok() => self.state.event(
5975 "land",
5976 format!("pushed rebased {} to {remote}", winner.branch),
5977 ),
5978 Ok(o) => {
5979 conflict = Some(format!(
5980 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5981 winner.branch,
5982 short(pinned),
5983 o.stderr.chars().take(600).collect::<String>()
5984 ));
5985 }
5986 Err(e) => {
5987 conflict = Some(format!(
5988 "rebased {} locally but could not push it: {e:#}",
5989 winner.branch
5990 ));
5991 }
5992 }
5993 }
5994 if let Some(why) = &conflict {
5995 self.state.status = RunStatus::Blocked;
5996 self.state.event("land", why.clone());
5997 }
5998 self.state.base_sync = Some(BaseSync {
5999 tip: tip.clone(),
6000 behind: 0,
6001 attempts,
6002 conflict,
6003 already_in: None,
6004 });
6005 self.state
6006 .event("land", format!("rebased {} onto {tracking}", winner.branch));
6007 }
6008 Ok(Some(conflict)) => {
6009 self.state.finish_base_operation(None);
6010 let why = format!(
6011 "{} conflicts with {tracking} and did not rebase: {}",
6012 winner.branch,
6013 conflict.chars().take(600).collect::<String>()
6014 );
6015 self.state.status = RunStatus::Blocked;
6016 self.state.base_sync = Some(BaseSync {
6017 tip,
6018 behind,
6019 attempts,
6020 conflict: Some(why.clone()),
6021 already_in: None,
6022 });
6023 self.state.event("land", why);
6024 }
6025 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6029 self.state.save()?;
6030 return Ok(());
6031 }
6032 Err(e) => {
6033 self.state.finish_base_operation(None);
6034 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6035 self.state.status = RunStatus::Blocked;
6036 self.state.base_sync = Some(BaseSync {
6037 tip,
6038 behind,
6039 attempts,
6040 conflict: Some(why.clone()),
6041 already_in: None,
6042 });
6043 self.state.event("land", why);
6044 }
6045 }
6046 if !self.state.base_sync_blocked() {
6047 self.state.base_sync_recovery.stop = None;
6048 }
6049 self.hold_base_sync_kind(stop_kind).await?;
6050 self.state.save()?;
6051 Ok(())
6052 }
6053
6054 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6060 if self.state.task_pr_branch().is_some() {
6063 return None;
6064 }
6065 let repo = self.state.repo.clone();
6066 let remote = self.state.config.merge.remote.clone();
6067 let fetched = git::fetch(&repo, &remote, branch).await;
6068 if !matches!(&fetched, Ok(o) if o.ok()) {
6069 return None;
6070 }
6071 let branch_tracking = format!("{remote}/{branch}");
6072 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6073 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6074 return None;
6075 }
6076 if !crate::reconcile::origin_missing(&repo, head, &theirs)
6077 .await
6078 .is_ok_and(|missing| missing.is_empty())
6079 {
6080 return Some(format!(
6081 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6082 them",
6083 short(&theirs)
6084 ));
6085 }
6086 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6087 Ok(o) if o.ok() => {
6088 self.state
6089 .event("land", format!("pushed rebased {branch} to {remote}"));
6090 None
6091 }
6092 Ok(o) => Some(format!(
6093 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6094 someone may have pushed): {}",
6095 short(&theirs),
6096 o.stderr.chars().take(600).collect::<String>()
6097 )),
6098 Err(e) => Some(format!(
6099 "{branch} is rebased locally but could not be pushed: {e:#}"
6100 )),
6101 }
6102 }
6103
6104 async fn settle_already_in(
6121 &mut self,
6122 branch: &str,
6123 tip: &str,
6124 head: &str,
6125 attempts: usize,
6126 behind: usize,
6127 ) -> Result<bool> {
6128 let repo = self.state.repo.clone();
6129 let remote = self.state.config.merge.remote.clone();
6130 let start = self.state.base_commit.clone();
6131 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6132 Ok(Some(e)) => e,
6133 Ok(None) => return Ok(false),
6134 Err(e) => {
6135 tracing::warn!("already-in-base check for {branch}: {e:#}");
6136 return Ok(false);
6137 }
6138 };
6139 let mut verified = vec![head.to_owned()];
6140 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6141 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6142 if matches!(&fetched, Ok(o) if o.ok())
6143 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6144 && theirs != head
6145 {
6146 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6147 Ok(Some(_)) => verified.push(theirs),
6148 _ => return Ok(false),
6149 }
6150 }
6151 let base_branch = self.state.base_branch.clone();
6152 let message = format!(
6153 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6154 land",
6155 evidence.names(),
6156 evidence.proof.as_str()
6157 );
6158 let closed =
6159 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6160 .await;
6161 match closed {
6162 Ok(Ok(url)) => self
6163 .state
6164 .event("land", format!("closed {url}: superseded on {base_branch}")),
6165 Ok(Err(why)) => self
6166 .state
6167 .event("land", format!("did not close a pull request: {why}")),
6168 Err(e) => {
6169 let why = format!(
6170 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6171 not be closed ({e:#}); resume to retry"
6172 );
6173 self.state.status = RunStatus::Blocked;
6174 self.state.base_sync = Some(BaseSync {
6175 tip: tip.to_owned(),
6176 behind,
6177 attempts,
6178 conflict: Some(why.clone()),
6179 already_in: None,
6180 });
6181 self.state.event("land", why);
6182 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6183 .await?;
6184 self.state.save()?;
6185 return Ok(true);
6186 }
6187 }
6188 self.state.status = RunStatus::AlreadyInBase;
6189 self.state.base_sync_recovery.stop = None;
6190 self.state.base_sync = Some(BaseSync {
6191 tip: tip.to_owned(),
6192 behind,
6193 attempts,
6194 conflict: None,
6195 already_in: Some(evidence),
6196 });
6197 self.state.event("land", message);
6198 self.state.save()?;
6199 self.settle_questions();
6200 Ok(true)
6201 }
6202
6203 fn landing_base(&self) -> String {
6213 self.state
6214 .base_sync
6215 .as_ref()
6216 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6217 }
6218
6219 pub async fn fix_selected(
6252 &mut self,
6253 ids: &[String],
6254 reason: &str,
6255 allow_stale: bool,
6256 ) -> Result<()> {
6257 let reason = reason.trim();
6258 if reason.is_empty() {
6259 bail!("a fix request needs a reason — that is the operator's own record of why");
6260 }
6261 if ids.is_empty() {
6262 bail!("no finding id given");
6263 }
6264 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6265 bail!(
6266 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6267 has already concluded — can be given a targeted fix. A run still \
6268 in progress should simply be resumed; a `merged` run's branch has \
6269 already landed, so its answer is a fresh `magi review <branch>`, \
6270 not reopening this run's own record",
6271 self.state.id,
6272 self.state.status.as_str()
6273 );
6274 }
6275 let _claim = FixClaim::acquire(&self.state.dir())?;
6278 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6279 let Some(winner) = self.state.winner().cloned() else {
6280 bail!("run {} has no winning candidate to fix", self.state.id);
6281 };
6282 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6283 bail!(
6284 "branch `{}` no longer exists; this run cannot be extended",
6285 winner.branch
6286 );
6287 }
6288 let home = crate::run::home();
6289 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6290 bail!(
6291 "run {} is currently being worked on by another magi process",
6292 self.state.id
6293 );
6294 }
6295 let mut seen = BTreeSet::new();
6299 let mut findings = Vec::new();
6300 let mut missing = Vec::new();
6301 for id in ids {
6302 if !seen.insert(id.clone()) {
6303 continue;
6304 }
6305 match self.state.finding(id) {
6306 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6307 id: f.id.clone(),
6308 severity: f.severity,
6309 reviewer_vote: rec.vote,
6310 round: round.round,
6311 round_head: round.head.clone(),
6312 reviewer: rec.reviewer,
6313 agent: rec.agent.clone(),
6314 file: f.file.clone(),
6315 line: f.line,
6316 title: f.title.clone(),
6317 detail: f.detail.clone(),
6318 outcome: OperatorFixOutcome::Pending,
6319 }),
6320 None => missing.push(id.clone()),
6321 }
6322 }
6323 if !missing.is_empty() {
6324 bail!(
6325 "unknown finding id(s): {}; nothing was changed",
6326 missing.join(", ")
6327 );
6328 }
6329
6330 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6331 let stale_details: Vec<(String, String)> = findings
6332 .iter()
6333 .filter(|f| f.round_head != head_at_request)
6334 .map(|f| (f.id.clone(), f.round_head.clone()))
6335 .collect();
6336 let stale = !stale_details.is_empty();
6337 if stale && !allow_stale {
6338 bail!(
6339 "the branch has moved since some finding(s) were raised — {} — now \
6340 at {}; pass --allow-stale to fix anyway, or re-run review first",
6341 stale_details
6342 .iter()
6343 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6344 .collect::<Vec<_>>()
6345 .join(", "),
6346 short(&head_at_request)
6347 );
6348 }
6349
6350 let request = OperatorFixRequest {
6351 requested_at: Timestamp::now(),
6352 reason: reason.to_owned(),
6353 findings,
6354 head_at_request: head_at_request.clone(),
6355 allow_stale,
6356 stale,
6357 fix: None,
6358 result_head: None,
6359 follow_up_review_run: None,
6360 };
6361 self.state.event(
6362 "fix",
6363 format!(
6364 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6365 request.findings.len(),
6366 request
6367 .findings
6368 .iter()
6369 .map(|f| f.id.as_str())
6370 .collect::<Vec<_>>()
6371 .join(", "),
6372 ),
6373 );
6374 self.state.operator_fixes.push(request);
6381 self.state.save()?;
6382 let request_index = self.state.operator_fixes.len() - 1;
6383
6384 let repo = self.state.repo.clone();
6393 let coordination = self.coordinate(&repo).await?;
6394 if winner.worktree.exists() {
6395 let dirty = git::git(
6398 &winner.worktree,
6399 &["status", "--porcelain", "--untracked-files=all"],
6400 )
6401 .await?;
6402 let only_withheld = dirty.lines().all(|l| {
6403 l.strip_prefix("?? ")
6404 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6405 });
6406 if !only_withheld {
6407 bail!(
6408 "`{}` has uncommitted changes; refusing to touch it — commit or \
6409 discard them first",
6410 winner.worktree.display()
6411 );
6412 }
6413 git::worktree_remove(&self.state.repo, &winner.worktree)
6414 .await
6415 .ok();
6416 }
6417 let fix_worktree = self.state.worktree_root().join("operator-fix");
6418 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6419 git::git(
6420 &self.state.repo,
6421 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6422 )
6423 .await
6424 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6425 if !git::is_clean(&fix_worktree).await? {
6426 git::worktree_remove(&self.state.repo, &fix_worktree)
6427 .await
6428 .ok();
6429 bail!(
6430 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6431 winner.branch
6432 );
6433 }
6434 let repo_for_setup = self.state.repo.clone();
6435 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6436 Ok(()) => {
6437 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6438 }
6439 Err(e) => Err(e),
6440 };
6441 if let Err(e) = setup {
6442 git::worktree_remove(&self.state.repo, &fix_worktree)
6443 .await
6444 .ok();
6445 return Err(e);
6446 }
6447
6448 self.state.resource_block = None;
6451
6452 let run_id = self.state.id.clone();
6453 let prompts = self.state.config.prompts.clone();
6454 let language = self.state.config.graph.language.clone();
6455 let sessions = self.state.config.graph.sessions;
6456 let artifacts = agent::artifacts_dir(&self.state.dir());
6457 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6458 .findings
6459 .iter()
6460 .map(|f| Finding {
6461 id: f.id.clone(),
6462 severity: f.severity,
6463 file: f.file.clone(),
6464 line: f.line,
6465 title: f.title.clone(),
6466 detail: f.detail.clone(),
6467 })
6468 .collect();
6469 let fix_prompt = prompt::operator_fix_in(
6470 &self.state.instruction,
6471 &finding_list,
6472 reason,
6473 &stale_details,
6474 &head_at_request,
6475 &language,
6476 &self.state.config.graph.github_language,
6477 );
6478 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6479 drop(coordination);
6480 let (job, seat, out) = self
6481 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6482 prompt: fix_prompt.clone(),
6483 spec,
6484 seat,
6485 cwd: fix_worktree.clone(),
6486 timeout,
6487 allow_write: true,
6488 sessions,
6489 artifacts: artifacts.clone(),
6490 stem: "operator-fix".to_owned(),
6491 handover: None,
6492 })
6493 .await;
6494 let agent_id = seat.agent.clone();
6495
6496 let mut fix = FixRecord {
6497 agent: agent_id,
6498 addressed: Vec::new(),
6499 rejected: Vec::new(),
6500 notes: String::new(),
6501 committed: false,
6502 failed: None,
6503 duration_ms: 0,
6504 continuation: None,
6505 };
6506 let mut final_seat = seat.clone();
6507 match out {
6508 AgentOutcome::Ok(o) => {
6509 fix.duration_ms = o.duration_ms;
6510 let parsed = verdict::extract_json::<FixReport>(&o.text);
6511 let incomplete_reason = match &parsed {
6512 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6513 "the reply parsed, but it reported a command whose own CLI \
6514 never confirmed an exit status"
6515 .to_owned(),
6516 ),
6517 Ok(_) => None,
6518 Err(e) => Some(e.to_string()),
6519 };
6520 match incomplete_reason {
6521 None => {
6522 let report = parsed.expect("checked Ok above");
6523 fix.addressed = report.addressed;
6524 fix.rejected = report.rejected;
6525 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6526 }
6527 Some(reason) => {
6528 let (resumed_seat, resolved, failure, cont) = self
6529 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6530 .await;
6531 fix.duration_ms += cont.cumulative_wait_ms;
6532 fix.continuation = Some(cont);
6533 final_seat = resumed_seat;
6534 match resolved {
6535 Some(report) => {
6536 fix.addressed = report.addressed;
6537 fix.rejected = report.rejected;
6538 fix.notes =
6539 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6540 }
6541 None => fix.failed = failure,
6542 }
6543 }
6544 }
6545 }
6546 AgentOutcome::Dropped(o) => {
6547 fix.duration_ms = o.duration_ms;
6548 let why = o
6549 .dropped
6550 .as_ref()
6551 .map(|d| d.why.as_str())
6552 .unwrap_or("the CLI ended the stream without delivering its answer");
6553 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6554 }
6555 AgentOutcome::Quota(o) => {
6556 self.state.quota.push(QuotaLoss::from_report(
6557 final_seat.key.clone(),
6558 "fix",
6559 o.quota.as_ref(),
6560 ));
6561 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6562 }
6563 AgentOutcome::Failed(e) => fix.failed = Some(e),
6564 AgentOutcome::ResourceBlocked(e) => {
6565 fix.failed = Some(format!("the fixer was not started: {e}"));
6566 }
6567 }
6568 if fix.continuation.is_none() {
6571 fix.continuation = Some(ContinuationRecord::not_needed());
6572 }
6573 self.state.seats.insert(final_seat.key.clone(), final_seat);
6574
6575 let rescue_message = format!(
6576 "magi: operator-selected fix ({}) (uncommitted work)",
6577 self.state.operator_fixes[request_index]
6578 .findings
6579 .iter()
6580 .map(|f| f.id.as_str())
6581 .collect::<Vec<_>>()
6582 .join(", ")
6583 );
6584 let repo = self.state.repo.clone();
6585 let coordination = self.coordinate(&repo).await?;
6586 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6587 self.state.note_withheld("fix", &r.withheld);
6588 }
6589 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6590 fix.committed = after != head_at_request;
6591 git::worktree_remove(&self.state.repo, &fix_worktree)
6592 .await
6593 .ok();
6594 drop(coordination);
6595
6596 self.state.event(
6597 "fix",
6598 match &fix.failed {
6599 Some(reason) => format!(
6600 "operator fix: adoption report was lost ({reason}); {}",
6601 if fix.committed {
6602 "committed"
6603 } else {
6604 "NO new commit"
6605 }
6606 ),
6607 None => format!(
6608 "operator fix: {} addressed, {} rejected, {}",
6609 fix.addressed.len(),
6610 fix.rejected.len(),
6611 if fix.committed {
6612 "committed"
6613 } else {
6614 "NO new commit"
6615 }
6616 ),
6617 },
6618 );
6619
6620 for f in &mut self.state.operator_fixes[request_index].findings {
6627 f.outcome = if fix.failed.is_some() {
6628 OperatorFixOutcome::Unreported
6629 } else if fix.addressed.contains(&f.id) {
6630 OperatorFixOutcome::Addressed
6631 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6632 OperatorFixOutcome::Rejected { why: r.why.clone() }
6633 } else {
6634 OperatorFixOutcome::Unreported
6635 };
6636 }
6637
6638 let committed = fix.committed;
6639 if committed {
6640 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6641 }
6642 self.state.operator_fixes[request_index].fix = Some(fix);
6643 self.state.save()?;
6646 if self.state.resource_block.is_some() {
6647 return self.stop_for_resource();
6648 }
6649
6650 if committed {
6651 self.state.event(
6652 "fix",
6653 format!(
6654 "operator fix committed {}; opening a follow-up review-only run",
6655 short(&after)
6656 ),
6657 );
6658 let origin =
6661 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6662 match Self::review(
6663 &self.state.repo,
6664 &winner.branch,
6665 self.state.config.clone(),
6666 origin,
6667 )
6668 .await
6669 {
6670 Ok(mut follow_up) => {
6671 follow_up.state.event(
6672 "start",
6673 format!(
6674 "requested by an operator fix on run {} for finding(s) {}",
6675 self.state.id,
6676 self.state.operator_fixes[request_index]
6677 .findings
6678 .iter()
6679 .map(|f| f.id.as_str())
6680 .collect::<Vec<_>>()
6681 .join(", "),
6682 ),
6683 );
6684 follow_up.state.save()?;
6685 let follow_up_id = follow_up.state.id.clone();
6686 let adopted = match crate::direct::adopt(
6689 &follow_up.state,
6690 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6691 ) {
6692 Ok(a) => a,
6693 Err(e) => {
6694 self.state.event(
6697 "fix",
6698 format!(
6699 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6700 ),
6701 );
6702 self.state.save()?;
6703 return Ok(());
6704 }
6705 };
6706 let executed = follow_up.execute().await;
6707 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6708 if let Some(a) = adopted {
6709 a.finish(&follow_up.state, executed);
6710 }
6711 if let Some(e) = failure {
6712 self.state.event(
6713 "fix",
6714 format!(
6715 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6716 ),
6717 );
6718 }
6719 self.state.operator_fixes[request_index].follow_up_review_run =
6720 Some(follow_up_id);
6721 }
6722 Err(e) => {
6723 self.state.event(
6724 "fix",
6725 format!("committed the fix but could not open a follow-up review: {e:#}"),
6726 );
6727 }
6728 }
6729 self.state.save()?;
6730 }
6731
6732 Ok(())
6733 }
6734
6735 async fn ask_fixer(
6755 &mut self,
6756 winner: &Candidate,
6757 node: &'static str,
6758 round: Option<usize>,
6759 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6760 ) -> (SeatJob, SeatState, AgentOutcome) {
6761 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6762 self.ask_fixer_among(attempts, node, round, None, build)
6763 .await
6764 }
6765
6766 async fn ask_fixer_among(
6771 &mut self,
6772 attempts: Vec<(AgentSpec, String)>,
6773 node: &'static str,
6774 round: Option<usize>,
6775 restore: Option<(&Path, &str)>,
6776 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6777 ) -> (SeatJob, SeatState, AgentOutcome) {
6778 let run_id = self.state.id.clone();
6779 let prompts = self.state.config.prompts.clone();
6780 let cache = self.state.config.cache_dir();
6781 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6782 let mut last = None;
6783 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6784 let seat = self.seat(&key, &spec.id);
6785 let mut job = build(spec.clone(), seat);
6786 if i > 0 {
6787 job.stem = format!("{}-{}", job.stem, spec.id);
6788 }
6789 let ctx = WaveCtx {
6790 carry_seats: false,
6791 pauses: Some((&self.pause, &self.interrupt)),
6792 run: &run_id,
6793 node,
6794 prompts: &prompts,
6795 cache: cache.as_deref(),
6796 round,
6797 };
6798 let (seat, out) =
6799 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6800 let advances = match &out {
6803 AgentOutcome::Ok(o) => agent::output_advances(o),
6804 AgentOutcome::ResourceBlocked(_) => false,
6805 _ => true,
6806 };
6807 if let Some(next) = ids.get(i + 1)
6808 && advances
6809 {
6810 self.state.seats.insert(seat.key.clone(), seat.clone());
6811 let class =
6812 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6813 let reason = match &out {
6814 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6815 other => fail_reason(other),
6816 };
6817 if let Some((wt, rev)) = restore
6821 && let Err(e) = self.graft_restore(wt, rev).await
6822 {
6823 let why =
6824 format!("could not restore the worktree before the next fixer: {e:#}");
6825 tracing::warn!("{why}");
6826 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6827 AgentOutcome::ResourceBlocked(why)
6828 } else {
6829 AgentOutcome::Failed(why)
6830 };
6831 last = Some((job, seat, out));
6832 break;
6833 }
6834 record_handover(
6835 &mut self.state,
6836 node,
6837 &seat.key,
6838 &spec.id,
6839 next,
6840 &class,
6841 &reason,
6842 );
6843 continue;
6844 }
6845 last = Some((job, seat, out));
6846 break;
6847 }
6848 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6849 if let AgentOutcome::Quota(output) = &out
6850 && node != "graft"
6851 {
6852 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6857 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6858 self.state.quota.push(loss);
6859 self.state.seats.insert(seat.key.clone(), seat.clone());
6860 self.state.status = RunStatus::Blocked;
6861 return (
6862 job,
6863 seat,
6864 AgentOutcome::ResourceBlocked(
6865 "fixer unavailable; restore agent availability and resume".into(),
6866 ),
6867 );
6868 }
6869 (job, seat, out)
6870 }
6871
6872 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6883 use crate::recovery::{Mode, Repair};
6884 if !self.state.reviews.is_empty() {
6885 return Ok(());
6886 }
6887 let Some(mut rec) = self.state.recovery.clone() else {
6888 return Ok(());
6889 };
6890 if rec.mode != Mode::VerifyRepair {
6891 return Ok(());
6892 }
6893 let wt = winner.worktree.clone();
6894 let mut head = git::rev_parse(&wt, "HEAD").await?;
6895 match rec.repair.clone() {
6896 Repair::Done { .. } => return Ok(()),
6897 Repair::WaitingForRestore { before } => {
6898 self.graft_restore(&wt, &before).await?;
6899 head = before;
6900 rec.repair = Repair::NotStarted;
6901 self.state.recovery = Some(rec.clone());
6902 self.state.save()?;
6903 }
6904 Repair::Started { before } => {
6905 if let Err(e) = self.graft_restore(&wt, &before).await {
6909 if e.is::<crate::repo_lock::ResourceBlocked>() {
6910 return Err(e);
6911 }
6912 return self
6913 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6914 .await;
6915 }
6916 rec.repair = Repair::Done {
6917 before: before.clone(),
6918 after: before,
6919 committed: false,
6920 failed: Some("interrupted; rolled back".to_owned()),
6921 };
6922 self.state
6923 .event("recovery", "the repair was interrupted and rolled back");
6924 self.state.recovery = Some(rec);
6925 self.state.save()?;
6926 return Ok(());
6927 }
6928 Repair::NotStarted => {}
6929 }
6930 if crate::recovery::recheck(&mut rec, &head) {
6931 self.state.event("recovery", rec.summary());
6932 self.state.recovery = Some(rec);
6933 self.state.save()?;
6934 return Ok(());
6935 }
6936 rec.repair = Repair::Started {
6937 before: head.clone(),
6938 };
6939 self.state.recovery = Some(rec.clone());
6940 self.state.save()?;
6941
6942 let language = self.state.config.graph.language.clone();
6943 let sessions = self.state.config.graph.sessions;
6944 let artifacts = agent::artifacts_dir(&self.state.dir());
6945 let fix_prompt = prompt::recovery_fix(
6946 &self.state.instruction,
6947 &rec,
6948 self.state.config.graph.review_rounds,
6949 &language,
6950 );
6951 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6952 self.state.event(
6953 "recovery",
6954 format!(
6955 "repair: the fixer starts from the failed verification on {}",
6956 short(&head)
6957 ),
6958 );
6959 let (_job, seat, out) = self
6960 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6961 prompt: fix_prompt.clone(),
6962 spec,
6963 seat,
6964 cwd: wt.clone(),
6965 timeout,
6966 allow_write: true,
6967 sessions,
6968 artifacts: artifacts.clone(),
6969 stem: "fix-recovery".to_owned(),
6970 handover: None,
6971 })
6972 .await;
6973 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6974 rec.repair = Repair::WaitingForRestore { before: head };
6975 self.state.recovery = Some(rec);
6976 return self.stop_for_resource();
6977 }
6978 let failed = match &out {
6979 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6980 AgentOutcome::Quota(o) => {
6981 self.state.quota.push(QuotaLoss::from_report(
6982 seat.key.clone(),
6983 "fix",
6984 o.quota.as_ref(),
6985 ));
6986 Some("rate limited (quota); the fixer could not run".to_owned())
6987 }
6988 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6989 AgentOutcome::Failed(e) => Some(e.clone()),
6990 };
6991 self.state.seats.insert(seat.key.clone(), seat);
6992 let coordination = self.coordinate(&wt).await?;
6993 let failed = match failed {
6994 Some(why) => {
6996 if let Err(e) = git::restore_to(&wt, &head).await {
6997 return self
6998 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6999 .await;
7000 }
7001 Some(why)
7002 }
7003 None => {
7004 if let Ok(r) = git::rescue_commit(
7005 &wt,
7006 "magi: recovery repair of the failed verification (uncommitted work)",
7007 )
7008 .await
7009 {
7010 self.state.note_withheld("fix", &r.withheld);
7011 }
7012 None
7013 }
7014 };
7015 let after = git::rev_parse(&wt, "HEAD").await?;
7016 drop(coordination);
7017 self.state.event(
7018 "recovery",
7019 match &failed {
7020 Some(why) => format!("repair: nothing kept ({why})"),
7021 None if after != head => {
7022 format!("repair: committed {}", short(&after))
7023 }
7024 None => "repair: the fixer made no commit".to_owned(),
7025 },
7026 );
7027 rec.repair = Repair::Done {
7028 before: head.clone(),
7029 committed: after != head,
7030 after,
7031 failed,
7032 };
7033 self.state.recovery = Some(rec);
7034 self.state.save()?;
7035 Ok(())
7036 }
7037
7038 async fn stop_recovery(
7041 &mut self,
7042 mut rec: crate::recovery::RecoveryRecord,
7043 head: &str,
7044 why: String,
7045 ) -> Result<()> {
7046 rec.repair = crate::recovery::Repair::Started {
7048 before: head.to_owned(),
7049 };
7050 self.state.recovery = Some(rec);
7051 self.state
7052 .event("recovery", format!("repair could not be undone: {why}"));
7053 self.state.status = RunStatus::Blocked;
7054 self.state.save()?;
7055 bail!("recovery repair could not be rolled back: {why}")
7056 }
7057
7058 async fn review_loop(&mut self) -> Result<()> {
7059 if self.hold_base_sync().await? {
7060 return Ok(());
7061 }
7062 loop {
7063 if self
7064 .state
7065 .base_sync
7066 .as_ref()
7067 .is_some_and(|s| s.conflict.is_some())
7068 {
7069 return Ok(());
7070 }
7071 if self.catchup_repairable() {
7072 let record = self.state.reviews.last().expect("repairable round").clone();
7073 let winner = self
7074 .state
7075 .winner()
7076 .cloned()
7077 .context("catch-up repair has no winner")?;
7078 if !self.fix_review_round(&winner, record).await? {
7079 return Ok(());
7080 }
7081 }
7082 self.review_loop_pass().await?;
7083 if !self.catchup_repairable() {
7084 return Ok(());
7085 }
7086 }
7087 }
7088
7089 fn catchup_repairable(&self) -> bool {
7092 self.state.reviews.len() < self.state.config.graph.review_rounds
7093 && self.state.reviews.last().is_some_and(|r| {
7094 !r.clean
7095 && r.blocking == 0
7096 && r.fix.is_none()
7097 && r.e2e_status() == E2eStatus::Failed
7098 })
7099 }
7100
7101 async fn review_loop_pass(&mut self) -> Result<()> {
7102 if self
7107 .state
7108 .base_sync
7109 .as_ref()
7110 .is_some_and(|s| s.conflict.is_some())
7111 {
7112 return Ok(());
7113 }
7114 let run_id = self.state.id.clone();
7119 let prompts = self.state.config.prompts.clone();
7120 let Some(winner) = self.state.winner().cloned() else {
7121 return Ok(());
7122 };
7123 let granted_review = self
7124 .state
7125 .repair_grants
7126 .last()
7127 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7128 .cloned();
7129 let fresh_review = granted_review
7130 .as_ref()
7131 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7132 let max_rounds = granted_review
7133 .as_ref()
7134 .map_or(self.state.config.graph.review_rounds, |g| {
7135 g.review_start.saturating_add(1)
7136 });
7137 if let Some(status) =
7147 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7148 {
7149 if status == RunStatus::Gating {
7152 self.record_contested_handoff();
7153 }
7154 self.state.status = status;
7155 self.state.save()?;
7156 return Ok(());
7157 }
7158 self.state.status = RunStatus::Reviewing;
7159 if !fresh_review
7169 && self.state.reviews.last().is_some_and(|r| {
7170 r.e2e_status() == E2eStatus::ResourceBlocked
7171 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7172 })
7173 {
7174 let shell = self.state.config.shell();
7175 return self
7176 .stop_unresolved_review(
7177 "the last round's own verification never resolved",
7178 &shell,
7179 &winner.worktree,
7180 )
7181 .await;
7182 }
7183
7184 if granted_review.is_none() {
7185 self.recovery_repair(&winner).await?;
7186 }
7187 if self.resource_halt()? {
7188 return Ok(());
7189 }
7190
7191 let repo = self.state.repo.clone();
7192 let root = self.state.worktree_root();
7193 let language = self.state.config.graph.language.clone();
7194 let sessions = self.state.config.graph.sessions;
7195 let artifacts = self.merge_repair_artifacts();
7196 let base = self.landing_base();
7197 let base_short = short(&base);
7198 let reviewers = self.roles.reviewers.clone();
7199 let shell = self.state.config.shell();
7200
7201 for round in (self.state.reviews.len() + 1)..=max_rounds {
7202 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7203 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7204 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7205 let prev_verification = self
7214 .state
7215 .reviews
7216 .last()
7217 .and_then(|r| r.verification_summary(&head));
7218 let prior_section = match (&self.state.recovery, round) {
7221 (Some(rec), 1) if self.state.reviews.is_empty() => {
7222 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7223 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7224 Some(crate::recovery::head_of(
7225 &git::diff(&winner.worktree, reviewed, "HEAD")
7226 .await
7227 .unwrap_or_default(),
7228 crate::recovery::SINCE_DIFF_MAX,
7229 ))
7230 }
7231 _ => None,
7232 };
7233 Some(prompt::recovery_review(rec, since.as_deref()))
7234 }
7235 _ => None,
7236 };
7237
7238 let mut jobs = Vec::new();
7242 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7243 self.coordinate(&repo).await?
7244 } else {
7245 None
7246 };
7247 for (r, spec) in reviewers.iter().cloned().enumerate() {
7248 let wt = root.join(format!("review-{}", r + 1));
7249 if wt.exists() {
7252 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7253 } else {
7254 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7255 None
7256 } else {
7257 self.coordinate(&repo).await?
7258 };
7259 git::worktree_add_detached(&repo, &wt, &head).await?;
7260 }
7261 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7262 None
7263 } else {
7264 self.coordinate(&repo).await?
7265 };
7266 self.setup_seat_worktree(&repo, &wt, false).await?;
7267 let seat_key = format!("review-{}", r + 1);
7268 let spec = pick_start_spec(
7271 &self.roles.reviewer_roster,
7272 spec,
7273 self.state.seat_history.get(&seat_key),
7274 );
7275 let seat = self.seat(&seat_key, &spec.id);
7276 jobs.push(SeatJob {
7277 prompt: prompt::review(&prompt::ReviewCtx {
7278 instruction: &self.state.instruction,
7279 branch: &winner.branch,
7280 base_short: &base_short,
7281 stat: &stat,
7282 patch: &patch,
7283 verification: prev_verification.as_ref(),
7284 prior: prior_section.as_deref(),
7285 reviewers: reviewers.len(),
7286 round,
7287 rounds: max_rounds,
7288 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7291 lens: Lens::for_seat(r),
7292 language: &language,
7293 github_language: &self.state.config.graph.github_language,
7294 }),
7295 spec,
7296 seat,
7297 cwd: wt,
7298 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7299 allow_write: false,
7300 sessions,
7301 artifacts: artifacts.clone(),
7302 stem: format!("review-{round}-{}", r + 1),
7303 handover: None,
7304 });
7305 }
7306
7307 drop(coordination);
7308 self.state.event(
7309 "review",
7310 format!(
7311 "round {round}: {} reviewers on {}",
7312 jobs.len(),
7313 short(&head)
7314 ),
7315 );
7316 let mut quota_losses = Vec::new();
7317 let review_retries = self.state.config.graph.retries;
7318 let review_cache = self.state.config.cache_dir();
7319 let ctx = WaveCtx {
7320 carry_seats: true,
7321 pauses: Some((&self.pause, &self.interrupt)),
7322 run: &run_id,
7323 node: "review",
7324 prompts: &prompts,
7325 cache: review_cache.as_deref(),
7326 round: Some(round),
7327 };
7328 let results = ask_json_wave::<Review>(
7329 jobs,
7330 Arc::clone(&self.sem),
7331 review_retries,
7332 &self.roles.reviewer_roster,
7333 &ctx,
7334 &mut quota_losses,
7335 &mut self.state,
7336 &|_: &Review| Ok(()),
7337 )
7338 .await;
7339 if self.state.resource_block.is_some() {
7340 return self.stop_for_resource();
7341 }
7342 let round_quota_missing = quota_losses.len();
7346 self.state.quota.extend(quota_losses);
7347
7348 let mut records = Vec::new();
7349 let mut all_findings = Vec::new();
7350 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7351 let agent_id = seat.agent.clone();
7352 self.state.seats.insert(seat.key.clone(), seat);
7353 let mut record = ReviewRecord {
7354 reviewer: r + 1,
7355 agent: agent_id,
7356 summary: String::new(),
7357 findings: Vec::new(),
7358 vote: None,
7359 failed: None,
7360 duration_ms: 0,
7361 attempts,
7367 };
7368 match res {
7369 Ok((review, out)) => {
7370 record.summary =
7378 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7379 record.vote = Some(review.vote);
7380 record.duration_ms = out.duration_ms;
7381 for (n, mut f) in review.findings.into_iter().enumerate() {
7382 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7385 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7386 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7387 f.file = f
7393 .file
7394 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7395 all_findings.push(f.clone());
7396 record.findings.push(f);
7397 }
7398 self.state.event(
7399 "review",
7400 format!(
7401 "round {round}: reviewer {} voted {} with {} finding(s)",
7402 r + 1,
7403 review.vote.label(),
7404 record.findings.len()
7405 ),
7406 );
7407 }
7408 Err(e) => {
7409 record.failed = Some(e.to_string());
7410 self.state.event(
7411 "review",
7412 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7413 );
7414 }
7415 }
7416 records.push(record);
7417 }
7418
7419 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7426 let vote_split =
7427 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7428 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7429 if vote_split {
7430 self.state.event(
7431 "review",
7432 format!(
7433 "round {round}: votes split ({}) — one round of reconsideration",
7434 initial_votes
7435 .iter()
7436 .map(|v| v.label())
7437 .collect::<Vec<_>>()
7438 .join(", ")
7439 ),
7440 );
7441 let panel: Vec<ReviewSeatReport<'_>> = records
7444 .iter()
7445 .filter_map(|r| {
7446 r.vote.map(|vote| ReviewSeatReport {
7447 reviewer: r.reviewer,
7448 vote,
7449 summary: &r.summary,
7450 findings: &r.findings,
7451 })
7452 })
7453 .collect();
7454
7455 let mut jobs = Vec::new();
7456 let mut seats_at = Vec::new();
7457 for (r, spec) in reviewers.iter().cloned().enumerate() {
7458 if records[r].vote.is_none() {
7462 continue;
7463 }
7464 let wt = root.join(format!("review-{}", r + 1));
7465 let seat_key = format!("review-{}", r + 1);
7466 let spec = self.occupant(&seat_key, spec);
7467 let seat = self.seat(&seat_key, &spec.id);
7468 let build = |with_patch: bool| {
7476 prompt::review_reconsider(&ReviewReconsiderCtx {
7477 instruction: &self.state.instruction,
7478 reviewer: r + 1,
7479 lens: Lens::for_seat(r),
7480 panel: &panel,
7481 patch: with_patch.then_some(ReviewPatch {
7482 branch: &winner.branch,
7483 base_short: &base_short,
7484 stat: &stat,
7485 patch: &patch,
7486 }),
7487 round,
7488 rounds: max_rounds,
7489 language: &language,
7490 })
7491 };
7492 let full = build(true);
7493 let prompt = if has_context(&spec, &seat, sessions) {
7494 build(false)
7495 } else {
7496 full.clone()
7497 };
7498 jobs.push(SeatJob {
7499 prompt,
7500 spec,
7501 seat,
7502 cwd: wt,
7503 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7504 allow_write: false,
7505 sessions,
7506 artifacts: artifacts.clone(),
7507 stem: format!("review-{round}-reconsider-{}", r + 1),
7508 handover: Some(full),
7509 });
7510 seats_at.push(r);
7511 }
7512
7513 let mut recon_quota_losses = Vec::new();
7514 let recon_cache = self.state.config.cache_dir();
7515 let recon_ctx = WaveCtx {
7516 carry_seats: true,
7517 pauses: Some((&self.pause, &self.interrupt)),
7518 run: &run_id,
7519 node: "review",
7520 prompts: &prompts,
7521 cache: recon_cache.as_deref(),
7522 round: Some(round),
7523 };
7524 let recon_results = ask_json_wave::<ReviewRevote>(
7525 jobs,
7526 Arc::clone(&self.sem),
7527 review_retries,
7528 &self.roles.reviewer_roster,
7529 &recon_ctx,
7530 &mut recon_quota_losses,
7531 &mut self.state,
7532 &|_: &ReviewRevote| Ok(()),
7533 )
7534 .await;
7535 if self.state.resource_block.is_some() {
7536 return self.stop_for_resource();
7537 }
7538 self.state.quota.extend(recon_quota_losses);
7539
7540 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7541 let agent_id = seat.agent.clone();
7542 self.state.seats.insert(seat.key.clone(), seat);
7543 let mut rec = ReviewRevoteRecord {
7544 reviewer: r + 1,
7545 agent: agent_id,
7546 vote: None,
7547 reason: String::new(),
7548 failed: None,
7549 };
7550 match res {
7551 Ok((rv, _)) => {
7552 rec.vote = Some(rv.vote);
7553 rec.reason =
7554 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7555 self.state.event(
7556 "review",
7557 format!(
7558 "round {round}: reviewer {} revoted {}",
7559 r + 1,
7560 rv.vote.label()
7561 ),
7562 );
7563 }
7564 Err(e) => {
7565 rec.failed = Some(e.to_string());
7566 self.state.event(
7567 "review",
7568 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7569 );
7570 }
7571 }
7572 reconsideration.push(rec);
7573 }
7574 } else if initial_votes.len() > 1 {
7575 self.state.event(
7576 "review",
7577 format!(
7578 "round {round}: votes agreed ({}) — no reconsideration",
7579 initial_votes[0].label()
7580 ),
7581 );
7582 }
7583
7584 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7585 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7586 let defer_e2e =
7597 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7598 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7599 let reason =
7600 format!("{blocking} blocking finding(s) already required a fix this round");
7601 self.state.event(
7602 "verify",
7603 format!(
7604 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7605 {}); it will run once a round has none left",
7606 short(&head)
7607 ),
7608 );
7609 (Vec::new(), false, true, Some(reason))
7610 } else {
7611 let e2e_commands = self.state.config.verify.e2e.clone();
7612 let cache_dir = self.state.config.cache_dir();
7613 let context = format!("round {round}");
7614 let wait = self.repository_wait();
7615 let (e2e, verify_retried) = with_cache_lease(
7616 &mut self.state,
7617 cache_dir.as_deref(),
7618 "e2e",
7619 "e2e",
7620 &winner.worktree,
7621 &head,
7622 verify_timeout,
7623 &context,
7624 &wait,
7625 |state, budget| {
7626 let shell = shell.clone();
7627 let e2e_commands = e2e_commands.clone();
7628 let worktree = winner.worktree.clone();
7629 let context = context.clone();
7630 async move {
7631 run_e2e_with_retry(
7632 state,
7633 &shell,
7634 &e2e_commands,
7635 &worktree,
7636 budget,
7637 &context,
7638 )
7639 .await
7640 }
7641 },
7642 )
7643 .await?;
7644 (e2e, verify_retried, false, None)
7645 };
7646
7647 let expected = records.len();
7648 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7649 let incomplete = answered < expected;
7650 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7651 let policy = self.state.config.graph.incomplete_review;
7652 let clean = round_is_clean(
7653 blocking,
7654 e2e_ok,
7655 answered,
7656 expected,
7657 round_quota_missing,
7658 policy,
7659 );
7660
7661 let mut round_record = ReviewRound {
7662 round,
7663 head: head.clone(),
7664 verified_head: None,
7665 verified_at: None,
7666 reviews: records,
7667 e2e,
7668 verify_retried,
7669 e2e_deferred,
7670 e2e_defer_reason,
7671 fix: None,
7672 blocking,
7673 answered,
7674 expected,
7675 clean,
7676 progressed: false,
7677 vote_split,
7678 reconsideration,
7679 verdict: None,
7680 };
7681 round_record.verdict = ReviewVote::worst(
7685 round_record
7686 .final_votes()
7687 .into_iter()
7688 .map(|(_, _, vote)| vote),
7689 );
7690 if !matches!(
7701 round_record.e2e_status(),
7702 E2eStatus::Deferred | E2eStatus::NotConfigured
7703 ) {
7704 round_record.verified_head = Some(head.clone());
7705 round_record.verified_at = Some(Timestamp::now());
7706 }
7707
7708 if incomplete {
7709 let missing: Vec<String> = round_record
7710 .reviews
7711 .iter()
7712 .filter(|r| r.failed.is_some())
7713 .map(|r| format!("review-{}", r.reviewer))
7714 .collect();
7715 self.state.event(
7716 "review",
7717 format!(
7718 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7719 missing.join(", ")
7720 ),
7721 );
7722 }
7723
7724 if clean {
7725 self.state.event(
7726 "review",
7727 if incomplete && policy == IncompleteReviewPolicy::Warn {
7728 format!(
7729 "round {round}: clean (warn policy, incomplete panel) — no \
7730 blocking findings from the seats that answered, verification green"
7731 )
7732 } else if incomplete {
7733 format!(
7734 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7735 quorum) — no blocking findings from the seats that answered, \
7736 verification green",
7737 expected - answered
7738 )
7739 } else {
7740 format!("round {round}: clean — no blocking findings, verification green")
7741 },
7742 );
7743 self.state.reviews.push(round_record);
7744 self.state.status = RunStatus::Gating;
7745 self.state.save()?;
7746 return Ok(());
7747 }
7748
7749 if incomplete && blocking == 0 && e2e_ok {
7757 self.state.reviews.push(round_record);
7758 self.state.save()?;
7759 if round == max_rounds {
7760 self.state.status = RunStatus::Blocked;
7761 self.state.event(
7762 "review",
7763 format!(
7764 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7765 refusing to call it clean",
7766 expected - answered
7767 ),
7768 );
7769 return Ok(());
7770 }
7771 continue;
7772 }
7773
7774 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7786 self.state.reviews.push(round_record);
7787 return self
7788 .stop_unresolved_review(
7789 "the round's own verification could not run",
7790 &shell,
7791 &winner.worktree,
7792 )
7793 .await;
7794 }
7795
7796 if round == max_rounds {
7797 self.state.reviews.push(round_record);
7798 return self
7799 .stop_reviewing(
7800 &format!(
7801 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7802 ),
7803 &shell,
7804 &winner.worktree,
7805 )
7806 .await;
7807 }
7808
7809 if !self.fix_review_round(&winner, round_record).await? {
7810 return Ok(());
7811 }
7812 }
7813 Ok(())
7814 }
7815
7816 async fn fix_review_round(
7819 &mut self,
7820 winner: &Candidate,
7821 mut round_record: ReviewRound,
7822 ) -> Result<bool> {
7823 let round = round_record.round;
7824 let shell = self.state.config.shell();
7825 let step = self.run_fix_round(winner, &mut round_record).await?;
7826 if matches!(step, FixStep::Resource) {
7827 self.stop_for_resource()?;
7828 return Ok(false);
7829 }
7830 if self.state.reviews.len() < round {
7831 self.state.reviews.push(round_record);
7832 } else {
7833 self.state.reviews[round - 1] = round_record;
7834 }
7835 self.state.save()?;
7836
7837 if matches!(step, FixStep::Recorded { report_lost: true }) {
7850 self.stop_reviewing(
7851 "the fixer's adoption report never came back, even after resuming its \
7852 own seat; refusing to start another round against the same worktree \
7853 while that is unresolved",
7854 &shell,
7855 &winner.worktree,
7856 )
7857 .await?;
7858 return Ok(false);
7859 }
7860
7861 let streak = self
7862 .state
7863 .reviews
7864 .iter()
7865 .rev()
7866 .take_while(|r| !r.progressed)
7867 .count();
7868 if streak >= STAGNANT_LIMIT {
7869 self.stop_reviewing(
7870 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7871 &shell,
7872 &winner.worktree,
7873 )
7874 .await?;
7875 return Ok(false);
7876 }
7877 Ok(true)
7878 }
7879
7880 async fn run_fix_round(
7882 &mut self,
7883 winner: &Candidate,
7884 round_record: &mut ReviewRound,
7885 ) -> Result<FixStep> {
7886 let round = round_record.round;
7887 let max_rounds = self.state.config.graph.review_rounds;
7888 let run_id = self.state.id.clone();
7889 let prompts = self.state.config.prompts.clone();
7890 let language = self.state.config.graph.language.clone();
7891 let sessions = self.state.config.graph.sessions;
7892 let artifacts = self.merge_repair_artifacts();
7893 let base = self.landing_base();
7894 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7895 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7896 let this_round_verification = round_record.verification_summary(&head);
7897 let all_findings: Vec<_> = round_record
7898 .reviews
7899 .iter()
7900 .flat_map(|r| r.findings.iter().cloned())
7901 .collect();
7902 let blocking_findings: Vec<_> = all_findings
7905 .iter()
7906 .filter(|f| f.severity.blocks())
7907 .cloned()
7908 .collect();
7909 let fix_prompt = prompt::fix_in(
7910 &self.state.instruction,
7911 &blocking_findings,
7912 this_round_verification.as_ref(),
7913 round,
7914 max_rounds,
7915 &language,
7916 &self.state.config.graph.github_language,
7917 );
7918 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7919 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7920 let (job, seat, out) = self
7921 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7922 prompt: fix_prompt.clone(),
7923 spec,
7924 seat,
7925 cwd: winner.worktree.clone(),
7926 timeout,
7927 allow_write: true,
7928 sessions,
7929 artifacts: artifacts.clone(),
7930 stem: format!("fix-{round}"),
7931 handover: None,
7932 })
7933 .await;
7934 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7937 return Ok(FixStep::Resource);
7938 }
7939 let agent_id = seat.agent.clone();
7940
7941 let mut fix = FixRecord {
7942 agent: agent_id,
7943 addressed: Vec::new(),
7944 rejected: Vec::new(),
7945 notes: String::new(),
7946 committed: false,
7947 failed: None,
7948 duration_ms: 0,
7949 continuation: None,
7950 };
7951 let mut continuation = ContinuationRecord::not_needed();
7952 let mut final_seat = seat.clone();
7953 match out {
7954 AgentOutcome::Ok(o) => {
7955 fix.duration_ms = o.duration_ms;
7956 let parsed = verdict::extract_json::<FixReport>(&o.text);
7957 let incomplete_reason = match &parsed {
7964 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7965 "the reply parsed, but it reported a command whose own CLI never \
7966 confirmed an exit status"
7967 .to_owned(),
7968 ),
7969 Ok(_) => None,
7970 Err(e) => Some(e.to_string()),
7971 };
7972 match incomplete_reason {
7973 None => {
7974 let report = parsed.expect("checked Ok above");
7975 fix.addressed = report.addressed;
7976 fix.rejected = report.rejected;
7977 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7978 }
7979 Some(reason) => {
7980 let (resumed_seat, resolved, failure, cont) = self
7981 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7982 .await;
7983 fix.duration_ms += cont.cumulative_wait_ms;
7984 continuation = cont;
7985 final_seat = resumed_seat;
7986 match resolved {
7987 Some(report) => {
7988 fix.addressed = report.addressed;
7989 fix.rejected = report.rejected;
7990 fix.notes =
7991 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7992 }
7993 None => fix.failed = failure,
7994 }
7995 }
7996 }
7997 }
7998 AgentOutcome::Dropped(o) => {
8000 fix.duration_ms = o.duration_ms;
8001 let why = o
8002 .dropped
8003 .as_ref()
8004 .map(|d| d.why.as_str())
8005 .unwrap_or("the CLI ended the stream without delivering its answer");
8006 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8007 }
8008 AgentOutcome::Quota(o) => {
8009 self.state.quota.push(QuotaLoss::from_report(
8010 final_seat.key.clone(),
8011 "fix",
8012 o.quota.as_ref(),
8013 ));
8014 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8015 }
8016 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8017 }
8018 if self.state.resource_block.is_some() {
8019 self.state.seats.insert(final_seat.key.clone(), final_seat);
8020 return Ok(FixStep::Resource);
8021 }
8022 fix.continuation = Some(continuation);
8023 self.state.seats.insert(final_seat.key.clone(), final_seat);
8024 let coordination = self.coordinate(&winner.worktree).await?;
8025 if let Ok(r) = git::rescue_commit(
8026 &winner.worktree,
8027 &format!("magi: review round {round} fixes (uncommitted work)"),
8028 )
8029 .await
8030 {
8031 self.state.note_withheld("fix", &r.withheld);
8032 }
8033 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8034 fix.committed = after != before;
8035 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8043 drop(coordination);
8044 let progressed = diff_after != patch;
8045 let commit_note = if fix.committed {
8046 "committed"
8047 } else {
8048 "NO new commit"
8049 };
8050 let tree_note = if progressed {
8051 "changed vs base"
8052 } else {
8053 "unchanged vs base"
8054 };
8055 self.state.event(
8056 "fix",
8057 match &fix.failed {
8058 Some(reason) => {
8064 format!(
8065 "round {round}: fixer's adoption report was lost ({reason}); \
8066 {commit_note}, tree {tree_note}"
8067 )
8068 }
8069 None => format!(
8070 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8071 {tree_note}{}",
8072 fix.addressed.len(),
8073 fix.rejected.len(),
8074 if continuation.outcome == ContinuationOutcome::Resumed {
8075 format!(
8076 " (adoption report recovered after {} continuation(s))",
8077 continuation.attempts
8078 )
8079 } else {
8080 String::new()
8081 },
8082 ),
8083 },
8084 );
8085 round_record.fix = Some(fix);
8086 round_record.progressed = progressed;
8087 Ok(FixStep::Recorded {
8088 report_lost: matches!(
8089 continuation.outcome,
8090 ContinuationOutcome::Exhausted
8091 | ContinuationOutcome::QuotaLost
8092 | ContinuationOutcome::NoSession
8093 ),
8094 })
8095 }
8096
8097 fn record_contested_handoff(&mut self) {
8131 if self.state.contested_handoff.is_some() {
8132 return;
8133 }
8134 let Some(contested) = self
8135 .state
8136 .reviews
8137 .last()
8138 .and_then(ReviewRound::contested_handoff)
8139 else {
8140 return;
8141 };
8142 self.state.event(
8143 "review",
8144 format!(
8145 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8146 wait for the owner's approval",
8147 contested.findings.len(),
8148 contested.rejecters.len()
8149 ),
8150 );
8151 self.state.contested_handoff = Some(contested);
8152 }
8153
8154 async fn stop_unresolved_review(
8155 &mut self,
8156 why: &str,
8157 shell: &[String],
8158 worktree: &Path,
8159 ) -> Result<()> {
8160 self.stop_reviewing_inner(why, shell, worktree, true).await
8161 }
8162
8163 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8164 self.stop_reviewing_inner(why, shell, worktree, false).await
8165 }
8166
8167 async fn stop_reviewing_inner(
8168 &mut self,
8169 why: &str,
8170 shell: &[String],
8171 worktree: &Path,
8172 repairable: bool,
8173 ) -> Result<()> {
8174 let round_idx = self.state.reviews.len() - 1;
8175 let needs_catchup_run = matches!(
8183 self.state.reviews[round_idx].e2e_status(),
8184 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8185 );
8186 if needs_catchup_run {
8187 let round = self.state.reviews[round_idx].round;
8188 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8189 let commands = self.state.config.verify.e2e.clone();
8190 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8191 let cache_dir = self.state.config.cache_dir();
8192 let context = format!(
8193 "round {round}: verification unresolved, catching up before the final decision"
8194 );
8195 let wait = self.repository_wait();
8196 let (outcomes, verify_retried) = with_cache_lease(
8197 &mut self.state,
8198 cache_dir.as_deref(),
8199 "e2e",
8200 "e2e",
8201 worktree,
8202 &attempted_head,
8203 timeout,
8204 &context,
8205 &wait,
8206 |state, budget| {
8207 let shell = shell.to_vec();
8208 let commands = commands.clone();
8209 let context = context.clone();
8210 async move {
8211 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8212 .await
8213 }
8214 },
8215 )
8216 .await?;
8217 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8218 self.state.record_base_progress(&attempted_head);
8219 }
8220 let last = &mut self.state.reviews[round_idx];
8221 last.e2e = outcomes;
8222 last.verify_retried = verify_retried;
8223 last.verified_head = Some(attempted_head);
8230 last.verified_at = Some(Timestamp::now());
8231 if verify_inconclusive(&last.e2e) {
8232 self.state.save()?;
8239 return Ok(());
8240 }
8241 last.e2e_deferred = false;
8242 if last.answered == last.expected {
8247 last.clean = round_is_clean(
8248 last.blocking,
8249 last.e2e.iter().all(CommandOutcome::ok),
8250 last.answered,
8251 last.expected,
8252 0,
8253 self.state.config.graph.incomplete_review,
8254 );
8255 }
8256 }
8257 let last = &self.state.reviews[round_idx];
8258 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8259
8260 match last.e2e_status() {
8261 E2eStatus::Failed => {
8262 let red: Vec<String> = last
8263 .e2e
8264 .iter()
8265 .filter(|o| !o.ok())
8266 .map(|o| {
8267 format!(
8268 "`{}` -> {:?}\n{}",
8269 o.command,
8270 o.code,
8271 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8272 )
8273 })
8274 .collect();
8275 let can_repair = last.blocking == 0 && last.fix.is_none();
8276 self.state.reviews[round_idx].clean = false;
8277 self.state
8278 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8279 self.state.status = RunStatus::Blocked;
8280 if repairable
8281 && can_repair
8282 && self.state.reviews.len() < self.state.config.graph.review_rounds
8283 {
8284 self.state.status = RunStatus::Reviewing;
8285 }
8286 }
8287 E2eStatus::ResourceBlocked => {
8292 self.state.event(
8293 "review",
8294 format!(
8295 "{why}; e2e could not run (resource unavailable); not \
8296 deciding yet"
8297 ),
8298 );
8299 }
8300 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8301 self.state.event(
8302 "review",
8303 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8304 );
8305 self.record_contested_handoff();
8306 self.state.status = RunStatus::Gating;
8307 }
8308 }
8309 self.state.save()?;
8310 Ok(())
8311 }
8312
8313 async fn gate(&mut self) -> Result<()> {
8316 if self.hold_base_sync().await? {
8317 return Ok(());
8318 }
8319 if self.state.status == RunStatus::Failed
8331 || self
8332 .state
8333 .base_sync
8334 .as_ref()
8335 .is_some_and(|s| s.conflict.is_some())
8336 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8337 != Some(RunStatus::Gating)
8338 {
8339 return Ok(());
8340 }
8341 let Some(winner) = self.state.winner().cloned() else {
8342 return Ok(());
8343 };
8344 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8345 self.state.reconcile_verification_head(&head);
8346 self.state.save()?;
8347 let wait = self.repository_wait();
8348 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8349 return Ok(());
8350 }
8351 if self.state.gate_ran {
8352 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8363 self.state.status = RunStatus::Blocked;
8364 self.state.save()?;
8365 }
8366 return Ok(());
8367 }
8368 self.state.status = RunStatus::Gating;
8369 let mut outcomes = self.run_gate(&winner).await?;
8370 let mut attempted_head = self.state.gate_head.clone();
8371 loop {
8372 if verify_inconclusive(&outcomes) {
8383 self.state.save()?;
8384 return Ok(());
8385 }
8386 if outcomes.iter().all(CommandOutcome::ok) {
8387 break;
8388 }
8389 let fixes_before = self.state.gate_fixes.len();
8390 match self.gate_fix_round(&winner, &outcomes).await? {
8391 GateFix::Retry => {
8392 outcomes = self.run_gate(&winner).await?;
8393 attempted_head = self.state.gate_head.clone();
8394 }
8395 GateFix::Stop => {
8396 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8397 if self.state.gate_fixes.len() > fixes_before
8398 && attempted_head.as_deref() != Some(current.as_str())
8399 && verify_current_head(&mut self.state, &winner, &wait).await?
8400 {
8401 outcomes = self.run_gate(&winner).await?;
8404 attempted_head = self.state.gate_head.clone();
8405 } else {
8406 break;
8407 }
8408 }
8409 GateFix::Defer => {
8410 self.state.save()?;
8411 return Ok(());
8412 }
8413 }
8414 }
8415 let passed = outcomes.iter().all(CommandOutcome::ok);
8416 self.state.gate = outcomes;
8417 self.state.gate_ran = true;
8418 self.state.gate_head = attempted_head;
8419 if !self.state.gate.is_empty()
8420 && !verify_inconclusive(&self.state.gate)
8421 && let Some(head) = self.state.gate_head.clone()
8422 {
8423 self.state.record_base_progress(&head);
8424 }
8425 self.state.save()?;
8426 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8427 return Ok(());
8428 }
8429 if !passed {
8430 self.state.status = RunStatus::Blocked;
8431 let spent = self.state.gate_fixes.len();
8432 self.state.event(
8433 "gate",
8434 if spent == 0 {
8435 "gate failed; not merging".to_owned()
8436 } else {
8437 format!("gate failed after {spent} gate-fix round(s); not merging")
8438 },
8439 );
8440 }
8441 self.state.save()?;
8442 Ok(())
8443 }
8444
8445 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8455 let commands = state.config.verify.pre_gate.clone();
8456 if commands.is_empty() {
8457 return;
8458 }
8459 let shell = state.config.shell();
8460 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8461 let (outcomes, _) = run_commands(
8462 state,
8463 "pre_gate",
8464 "pre_gate",
8465 0,
8466 &shell,
8467 &commands,
8468 &winner.worktree,
8469 timeout,
8470 )
8471 .await;
8472 for o in &outcomes {
8473 if !o.ok() {
8474 tracing::warn!(
8475 "pre_gate `{}` failed ({:?}); the gate decides",
8476 o.command,
8477 o.code
8478 );
8479 }
8480 state.event(
8481 "pre_gate",
8482 format!(
8483 "`{}` -> {}",
8484 o.command,
8485 if o.ok() {
8486 "pass".to_owned()
8487 } else {
8488 format!(
8489 "FAIL ({:?})\n{}",
8490 o.code,
8491 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8492 )
8493 }
8494 ),
8495 );
8496 }
8497 state.pre_gate = outcomes;
8498 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8499 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8500 Ok(head) => {
8501 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8502 state.pre_gate_commit = Some(head);
8503 }
8504 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8505 },
8506 Ok(false) => {}
8507 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8508 }
8509 if let Err(e) = state.save() {
8510 tracing::warn!("could not persist the pre_gate record: {e:#}");
8511 }
8512 }
8513
8514 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8517 let wait = self.repository_wait();
8518 run_landing_gate(&mut self.state, winner, &wait).await
8519 }
8520
8521 async fn gate_fix_round(
8534 &mut self,
8535 winner: &Candidate,
8536 outcomes: &[CommandOutcome],
8537 ) -> Result<GateFix> {
8538 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8539 g.phase == crate::repair_budget::Phase::Fixing
8540 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8541 }) {
8542 self.state.gate_fixes.len().saturating_add(1)
8543 } else {
8544 self.state.config.graph.gate_fix_rounds
8545 };
8546 let spent = self.state.gate_fixes.len();
8547 if spent >= cap {
8548 if cap > 0 {
8549 self.state.event(
8550 "gate",
8551 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8552 );
8553 }
8554 return Ok(GateFix::Stop);
8555 }
8556 if !gate_fixable(outcomes) {
8557 self.state.event(
8558 "gate",
8559 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8560 command or similar); not spending a fix round on it",
8561 );
8562 return Ok(GateFix::Stop);
8563 }
8564 let min_free = self.state.config.disk.min_free_bytes;
8565 if min_free > 0 {
8566 match crate::disk::free_bytes(&winner.worktree) {
8567 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8568 Ok(free) => {
8569 self.state.event(
8570 "gate",
8571 format!(
8572 "only {free} bytes free ({min_free} required by `[disk] \
8573 min_free_bytes`); not spending a fix round on a failure the disk \
8574 may explain"
8575 ),
8576 );
8577 return Ok(GateFix::Stop);
8578 }
8579 Err(e) => {
8580 self.state.event(
8581 "gate",
8582 format!("free disk space could not be measured ({e:#}); no fix round"),
8583 );
8584 return Ok(GateFix::Stop);
8585 }
8586 }
8587 }
8588
8589 let attempt = spent + 1;
8590 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8591 let base = self.landing_base();
8592 let fix_prompt = prompt::gate_fix_in(
8593 &self.state.instruction,
8594 &failed,
8595 attempt,
8596 cap,
8597 &self.state.config.graph.language,
8598 &self.state.config.graph.github_language,
8599 );
8600 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8601 let sessions = self.state.config.graph.sessions;
8602 let artifacts = self.merge_repair_artifacts();
8603 self.state.event(
8604 "gate",
8605 format!("gate failed; gate-fix round {attempt} of {cap}"),
8606 );
8607 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8608 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8609 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8612 head: before.clone(),
8613 passed: false,
8614 outcomes: Vec::new(),
8615 });
8616 self.state.save()?;
8617 let (_, seat, out) = self
8618 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8619 prompt: fix_prompt.clone(),
8620 spec,
8621 seat,
8622 cwd: winner.worktree.clone(),
8623 timeout,
8624 allow_write: true,
8625 sessions,
8626 artifacts: artifacts.clone(),
8627 stem: format!("gate-fix-{attempt}"),
8628 handover: None,
8629 })
8630 .await;
8631 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8634 return Ok(GateFix::Defer);
8635 }
8636 let mut record = GateFixRecord {
8637 agent: seat.agent.clone(),
8638 failed,
8639 notes: String::new(),
8640 committed: false,
8641 error: None,
8642 };
8643 match out {
8644 AgentOutcome::Ok(o) => {
8645 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8648 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8649 }
8650 }
8651 AgentOutcome::Dropped(_) => {
8652 record.error = Some("the CLI dropped the stream".to_owned());
8653 }
8654 AgentOutcome::Quota(o) => {
8655 self.state.quota.push(QuotaLoss::from_report(
8656 seat.key.clone(),
8657 "gate-fix",
8658 o.quota.as_ref(),
8659 ));
8660 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8661 }
8662 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8663 }
8664 self.state.seats.insert(seat.key.clone(), seat);
8665 let coordination = self.coordinate(&winner.worktree).await?;
8666 if let Ok(r) = git::rescue_commit(
8667 &winner.worktree,
8668 &format!("magi: gate fix {attempt} (uncommitted work)"),
8669 )
8670 .await
8671 {
8672 self.state.note_withheld("gate-fix", &r.withheld);
8673 }
8674 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8675 record.committed = after != before;
8676 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8677 let note = record.error.clone();
8678 self.state.gate_fixes.push(record);
8679 self.state.save()?;
8680 if !changed {
8681 self.state.event(
8682 "gate",
8683 match note {
8684 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8685 None => format!("gate-fix round {attempt}: the tree did not change"),
8686 },
8687 );
8688 return Ok(GateFix::Stop);
8689 }
8690 self.state.event(
8691 "gate",
8692 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8693 );
8694
8695 drop(coordination);
8696 let wait = self.repository_wait();
8697 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8698 }
8699
8700 async fn guarded_pr_message(
8707 &mut self,
8708 winner: &Candidate,
8709 facts: Option<&BranchFacts>,
8710 posting: bool,
8711 ) -> Result<Option<PrMessage>> {
8712 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8713 let head = match final_head.as_deref() {
8714 Some(h) => crate::pr_outcome::FinalHead::At(h),
8715 None => crate::pr_outcome::FinalHead::Unknown,
8716 };
8717 let summarized = if posting {
8719 Some(self.summarize_change(winner, final_head.as_deref()).await)
8720 } else {
8721 None
8722 };
8723 if self.state.resource_block.is_some() {
8728 self.state.event(
8729 "pr-summary",
8730 "a shared resource was busy; nothing was published, resume to summarize again",
8731 );
8732 return Ok(None);
8733 }
8734 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8735 anyhow::bail!(
8736 "the final commit moved during public description generation; retry before publishing"
8737 );
8738 }
8739 let note = match (&summarized, &self.state.change_summary) {
8740 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8741 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8742 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8743 (None, _) => ChangeNote::NotAsked,
8744 };
8745 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8746 if !posting {
8747 let identity = crate::scrub::Identity::current();
8748 return Ok(Some(PrMessage {
8749 title: crate::scrub::scrub(&pr.title, &identity),
8750 body: crate::scrub::scrub(&pr.body, &identity),
8751 }));
8752 }
8753 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8756 if let Some(known) = self
8757 .state
8758 .github_text
8759 .clone()
8760 .filter(|g| g.fingerprint == raw_fp)
8761 {
8762 return self.settle_withheld(winner, &pr, known).await;
8763 }
8764 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8765 let mut violations =
8766 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8767 let identity = crate::scrub::Identity::current();
8768 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8769 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8770 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8771 {
8772 violations.push(crate::github_text::Violation::SensitiveData);
8773 }
8774 let has_language = violations
8778 .iter()
8779 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8780 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8781 self.state.event(
8782 "github-text",
8783 if has_language {
8784 format!("description rejected: {violations:?}; requesting one rewrite")
8785 } else {
8786 format!("description rejected: {violations:?}; asking the owner")
8787 },
8788 );
8789 let mut rewritten = false;
8790 let mut rejected_rewrite = None;
8791 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8792 {
8793 let output = text;
8794 if let Some(title) = summary_title(&output) {
8795 let raw_body = summary_without_title(&output);
8798 let clean_title = crate::scrub::scrub(&title, &identity);
8799 let mut body = crate::scrub::scrub(&raw_body, &identity);
8800 let redacted = clean_title != title || body != raw_body;
8801 let title = clean_title;
8802 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8803 if !complete {
8804 self.state.event(
8805 "github-text",
8806 "description rewrite omitted required findings",
8807 );
8808 }
8809 if !body.trim().is_empty() {
8810 body.push_str("\n\n");
8811 body.push_str(&public_pr_evidence(&self.state, head));
8812 }
8813 let marker = format!("magi:run/{}", self.state.id);
8814 if !body.trim().is_empty() && !body.contains(&marker) {
8815 body.push_str(&format!("\n\n{marker}"));
8816 }
8817 let rewrite = PrMessage {
8818 title: title.clone(),
8819 body: body.clone(),
8820 };
8821 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8822 let mut failed = crate::github_text::check_for(
8823 &self.state.config.graph,
8824 &title,
8825 &body,
8826 redecision,
8827 );
8828 if (!crate::github_text::shareable(&title)
8829 || !crate::github_text::shareable(&body))
8830 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8831 {
8832 failed.push(crate::github_text::Violation::SensitiveData);
8833 }
8834 if complete
8835 && !title.trim().is_empty()
8836 && !body.trim().is_empty()
8837 && !failed.is_empty()
8838 {
8839 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8840 title: title.clone(),
8841 body: body.clone(),
8842 failed: failed
8843 .iter()
8844 .map(|v| crate::github_text::category(*v).to_owned())
8845 .collect(),
8846 });
8847 }
8848 if complete
8849 && !title.trim().is_empty()
8850 && !raw_body.trim().is_empty()
8851 && failed.is_empty()
8852 {
8853 decision = redecision;
8854 pr = PrMessage { title, body };
8855 rewritten = true;
8856 self.state
8857 .event("github-text", "description rewrite passed");
8858 if redacted {
8859 self.state.event(
8860 "github-text",
8861 "sensitive data removed from description rewrite",
8862 );
8863 }
8864 }
8865 }
8866 }
8867 if !rewritten {
8868 self.state.event(
8869 "github-text",
8870 "description rewrite unavailable or rejected; asking the owner",
8871 );
8872 if let Some(w) =
8873 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8874 {
8875 self.retire_github_text_question("the withheld text changed");
8878 let record = crate::run::GithubTextAsk {
8879 rewrite: rejected_rewrite,
8880 fingerprint: raw_fp,
8881 title: w.title,
8882 body: w.body,
8883 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8884 question: None,
8885 asks: 0,
8886 resolved: false,
8887 chosen_title: None,
8888 choice: None,
8889 publication: None,
8890 };
8891 return self.settle_withheld(winner, &pr, record).await;
8892 }
8893 }
8894 }
8895 let (title, body) =
8896 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8897 Ok(Some(PrMessage { title, body }))
8898 }
8899
8900 async fn summarize_change(
8911 &mut self,
8912 winner: &Candidate,
8913 head: Option<&str>,
8914 ) -> std::result::Result<(), String> {
8915 let Some(head) = head else {
8916 return Err("the final commit could not be read".to_owned());
8917 };
8918 let remote = self.state.config.merge.remote.clone();
8919 let tracking = format!("{remote}/{}", self.state.base_branch);
8920 if !git::rev_exists(&winner.worktree, &tracking).await {
8921 return Err(format!("the remote base `{tracking}` could not be read"));
8922 }
8923 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8924 Ok(b) if !b.is_empty() => b,
8925 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8926 };
8927 let evidence = crate::pr_outcome::summary_evidence(
8928 &self.state,
8929 crate::pr_outcome::FinalHead::At(head),
8930 );
8931 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8932 if self
8933 .state
8934 .change_summary
8935 .as_ref()
8936 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8937 {
8938 return Ok(());
8939 }
8940 if self.state.change_summary.take().is_some() {
8941 self.state.event(
8942 "pr-summary",
8943 "the final diff or evidence changed; the earlier change summary was discarded",
8944 );
8945 }
8946 if let Err(e) = write_artifact(
8948 &self.state,
8949 &format!("public-description-{fingerprint}.md"),
8950 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8951 ) {
8952 tracing::warn!("could not save public description evidence: {e:#}");
8953 }
8954 let chain = match agent::pick_chain(
8955 &self.state.config.agents,
8956 self.state.config.roles.synthesizer.as_ref(),
8957 &agent::installed,
8958 "synthesizer",
8959 ) {
8960 Ok(chain) => chain,
8961 Err(e) => {
8962 tracing::warn!("no agent for the change summary: {e:#}");
8963 self.state.event("pr-summary", "no summary agent available");
8964 return Err("no agent was available to summarize it".to_owned());
8965 }
8966 };
8967 let subjects = git::subjects(&winner.worktree, &base, head)
8968 .await
8969 .unwrap_or_default();
8970 let stat = git::diff_stat(&winner.worktree, &base, head)
8971 .await
8972 .unwrap_or_default();
8973 let stat: String = stat.chars().take(6_000).collect();
8974 let mut prompt = crate::prompt::change_summary_in(
8975 &base,
8976 head,
8977 &subjects,
8978 stat.trim(),
8979 &self.state.config.graph.github_language,
8980 );
8981 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8982 let sessions = self.state.config.graph.sessions;
8983 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8984 let prompts = self.state.config.prompts.clone();
8985 let cache = self.state.config.cache_dir();
8986 let run = self.state.id.clone();
8987 for (i, spec) in chain.into_iter().enumerate() {
8988 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8989 let github_spec = match agent::for_github_text(
8990 &spec,
8991 &self.state.config.graph.github_language,
8992 &winner.worktree,
8993 ) {
8994 Ok(spec) => spec,
8995 Err(e) => {
8996 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8997 continue;
8998 }
8999 };
9000 let job = SeatJob {
9001 spec: github_spec,
9002 seat,
9003 cwd: winner.worktree.clone(),
9004 prompt: prompt.clone(),
9005 timeout,
9006 allow_write: false,
9007 sessions,
9008 artifacts: agent::artifacts_dir(&self.state.dir()),
9009 stem: if i == 0 {
9010 "change-summary".to_owned()
9011 } else {
9012 format!("change-summary-{}", spec.id)
9013 },
9014 handover: None,
9015 };
9016 let ctx = WaveCtx {
9017 run: &run,
9018 node: "pr-summary",
9019 prompts: &prompts,
9020 cache: cache.as_deref(),
9021 round: None,
9022 carry_seats: false,
9023 pauses: Some((&self.pause, &self.interrupt)),
9024 };
9025 let (seat, outcome) =
9026 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9027 self.state.seats.insert(seat.key.clone(), seat);
9028 let text = match outcome {
9029 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9030 AgentOutcome::ResourceBlocked(why) => {
9031 if self.state.resource_block.is_none() {
9034 self.state.resource_block = Some(crate::run::ResourceBlock {
9035 cause: crate::run::ResourceBlockCause::Other,
9036 node: "pr-summary".to_owned(),
9037 kind: "shared resource".to_owned(),
9038 detail: why,
9039 at: Timestamp::now(),
9040 });
9041 }
9042 return Err("a shared resource was busy".to_owned());
9043 }
9044 _ => continue,
9045 };
9046 let blind = &self.state.config.blind;
9047 let Some(title) = summary_title(&text) else {
9048 continue;
9049 };
9050 let body: String = summary_without_title(&text)
9051 .lines()
9052 .filter(|l| !l.contains("magi:run/"))
9053 .collect::<Vec<_>>()
9054 .join("\n");
9055 let body = blind::sanitize_prose(body.trim(), blind);
9056 let title = blind::sanitize_prose(&title, blind);
9057 if body.trim().is_empty() || title.trim().is_empty() {
9058 continue;
9059 }
9060 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9061 continue;
9062 }
9063 if git::rev_parse(&winner.worktree, "HEAD")
9065 .await
9066 .ok()
9067 .as_deref()
9068 != Some(head)
9069 {
9070 return Err("the branch moved while the summary was written".to_owned());
9071 }
9072 if git::merge_base(&winner.worktree, &tracking, head)
9073 .await
9074 .ok()
9075 .as_deref()
9076 != Some(base.as_str())
9077 || crate::pr_outcome::summary_evidence(
9078 &self.state,
9079 crate::pr_outcome::FinalHead::At(head),
9080 ) != evidence
9081 {
9082 return Err("the summary evidence changed while it was written".to_owned());
9083 }
9084 self.state.change_summary = Some(crate::run::ChangeSummary {
9085 head: head.to_owned(),
9086 base,
9087 evidence: fingerprint,
9088 title,
9089 body,
9090 });
9091 self.state.event(
9092 "pr-summary",
9093 format!("change summary written by {} for the final diff", spec.id),
9094 );
9095 if let Err(e) = self.state.save() {
9096 tracing::warn!("could not save the change summary: {e:#}");
9097 }
9098 return Ok(());
9099 }
9100 self.state
9101 .event("pr-summary", "no agent produced a change summary");
9102 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9103 }
9104
9105 async fn rewrite_pr_text(
9115 &mut self,
9116 winner: &Candidate,
9117 pr: &PrMessage,
9118 violations: &[crate::github_text::Violation],
9119 ) -> Option<String> {
9120 let prompt = format!(
9121 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9122 pr.title, pr.body
9123 );
9124 let sessions = self.state.config.graph.sessions;
9125 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9126 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9129 let key = format!("impl-{}", winner.label);
9130 if let Some(spec) = self
9131 .state
9132 .config
9133 .agents
9134 .iter()
9135 .find(|a| a.id == winner.agent)
9136 .cloned()
9137 && let Some(existing) = self.state.seats.get(&key)
9138 && existing.agent == spec.id
9139 && agent::has_session(spec.kind, existing, sessions)
9140 {
9141 attempts.push((spec, Some(existing.clone())));
9142 }
9143 match agent::pick_chain(
9144 &self.state.config.agents,
9145 self.state.config.roles.language_judge.as_ref(),
9146 &agent::installed,
9147 "language_judge",
9148 ) {
9149 Ok(chain) => {
9150 for spec in chain {
9151 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9152 continue;
9153 }
9154 attempts.push((spec, None));
9155 }
9156 }
9157 Err(e) if attempts.is_empty() => {
9158 tracing::warn!("no agent for the description rewrite: {e:#}");
9159 self.state
9160 .event("github-text", "no rewrite agent available");
9161 return None;
9162 }
9163 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9164 }
9165 let prompts = self.state.config.prompts.clone();
9166 let cache = self.state.config.cache_dir();
9167 let run = self.state.id.clone();
9168 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9169 let resumed = resumed_seat.is_some();
9170 let seat = resumed_seat.unwrap_or_else(|| {
9171 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9172 });
9173 let github_spec = match agent::for_github_text(
9174 &spec,
9175 &self.state.config.graph.github_language,
9176 &winner.worktree,
9177 ) {
9178 Ok(spec) => spec,
9179 Err(e) => {
9180 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9181 continue;
9182 }
9183 };
9184 let job = SeatJob {
9185 spec: github_spec,
9186 seat,
9187 cwd: winner.worktree.clone(),
9188 prompt: prompt.clone(),
9189 timeout: retry_budget(timeout_full, resumed),
9190 allow_write: false,
9191 sessions,
9192 artifacts: agent::artifacts_dir(&self.state.dir()),
9193 stem: if i == 0 {
9194 "github-text-rewrite".to_owned()
9195 } else {
9196 format!("github-text-rewrite-{}", spec.id)
9197 },
9198 handover: None,
9199 };
9200 let ctx = WaveCtx {
9201 run: &run,
9202 node: "github-text",
9203 prompts: &prompts,
9204 cache: cache.as_deref(),
9205 round: None,
9206 carry_seats: false,
9207 pauses: Some((&self.pause, &self.interrupt)),
9208 };
9209 let (seat, outcome) =
9210 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9211 self.state.seats.insert(seat.key.clone(), seat);
9212 match outcome {
9213 AgentOutcome::Ok(o)
9214 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9215 {
9216 return Some(o.text);
9217 }
9218 AgentOutcome::ResourceBlocked(_) => return None,
9219 _ => {}
9220 }
9221 }
9222 None
9223 }
9224
9225 fn retire_github_text_question(&mut self, why: &str) {
9227 let Some(id) = self
9228 .state
9229 .github_text
9230 .as_ref()
9231 .and_then(|g| g.question.clone())
9232 else {
9233 return;
9234 };
9235 let _ = ask::Questions::open().update(&id, |q| {
9236 q.abandon(why);
9237 Ok(())
9238 });
9239 }
9240
9241 async fn settle_withheld(
9245 &mut self,
9246 winner: &Candidate,
9247 pr: &PrMessage,
9248 mut g: crate::run::GithubTextAsk,
9249 ) -> Result<Option<PrMessage>> {
9250 use crate::github_text as gt;
9251 let store = ask::Questions::open();
9252 let timeout = self.state.config.graph.answer_timeout;
9253 if !g.resolved {
9254 let mut standing = g
9255 .question
9256 .as_ref()
9257 .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9258 if let Some(q) = standing.as_ref()
9259 && q.status.open()
9260 && !store.held_reply(&q.id)
9261 && gt::expired(q, timeout)
9262 {
9263 let why = format!("no answer within {}s of asking", timeout.max(1));
9264 standing = store
9265 .update(&q.id, |q| {
9266 q.abandon(&why);
9267 Ok(())
9268 })
9269 .ok()
9270 .map(|(q, ())| q);
9271 }
9272 let mut ask_again: Option<Option<Vec<String>>> = None;
9273 match standing {
9274 None if g.asks <= 1 => {
9277 g.asks = 0;
9278 ask_again = Some(None);
9279 }
9280 None => {
9281 g.resolved = true;
9282 g.chosen_title = None;
9283 g.choice = Some(GithubTextChoice::Fallback);
9284 }
9285 Some(q) if q.status.open() => {
9286 self.park_on_github_text(&g)?;
9287 return Ok(None);
9288 }
9289 Some(q) => match gt::read_reply(&q) {
9290 gt::Reply::Fallback => {
9291 g.resolved = true;
9292 g.chosen_title = None;
9293 g.choice = Some(GithubTextChoice::Fallback);
9294 }
9295 gt::Reply::Original => {
9296 g.resolved = true;
9297 g.chosen_title = None;
9298 g.choice = Some(GithubTextChoice::Original);
9299 }
9300 gt::Reply::Rewrite => {
9301 g.resolved = true;
9302 g.chosen_title = None;
9303 g.choice = Some(if g.rewrite.is_some() {
9304 GithubTextChoice::Rewrite
9305 } else {
9306 GithubTextChoice::Fallback
9307 });
9308 }
9309 gt::Reply::Title(text) => {
9310 let first = text
9311 .lines()
9312 .map(str::trim)
9313 .find(|l| !l.is_empty())
9314 .unwrap_or("");
9315 let decision = if first.is_empty() {
9316 None
9317 } else {
9318 let candidate = PrMessage {
9319 title: first.to_owned(),
9320 body: String::new(),
9321 };
9322 self.judge_pr_language(&winner.worktree, &candidate).await
9323 };
9324 match gt::vet_title(
9325 &text,
9326 decision,
9327 self.state.config.graph.github_english(),
9328 ) {
9329 Ok(title) => {
9330 g.resolved = true;
9331 g.chosen_title = Some(title);
9332 g.choice = Some(GithubTextChoice::MyTitle);
9333 }
9334 Err(cats) if g.asks < 2 => {
9335 ask_again = Some(Some(
9336 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9337 ))
9338 }
9339 Err(_) => {
9340 g.resolved = true;
9341 g.chosen_title = None;
9342 g.choice = Some(GithubTextChoice::Fallback);
9343 }
9344 }
9345 }
9346 },
9347 }
9348 if let Some(rejected) = ask_again {
9349 let w = gt::Withheld {
9352 title: g.title,
9353 body: g.body,
9354 categories: g.categories.clone(),
9355 };
9356 let artifact = match crate::run::write_artifact(
9359 &self.state,
9360 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9361 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9362 ) {
9363 Ok(p) => Some(p),
9364 Err(e) => {
9365 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9366 None
9367 }
9368 };
9369 let (detail, comparison) = gt::question_content(
9370 &w,
9371 &pr.title,
9372 &pr.body,
9373 &self.state.id,
9374 rejected.as_deref(),
9375 artifact.as_deref(),
9376 g.rewrite.as_ref(),
9377 );
9378 let mut q = ask::Question::new(
9379 self.state.id.clone(),
9380 gt::ASK_NODE.to_owned(),
9381 gt::ASK_SEAT.to_owned(),
9382 gt::question_summary(&self.state.config.graph.language, &w),
9383 detail,
9384 gt::question_choices(&w, g.rewrite.as_ref()),
9385 );
9386 q.posting_comparison = Some(comparison);
9387 q.answer_timeout = timeout;
9388 g.question = Some(q.id.clone());
9391 g.asks += 1;
9392 self.state.github_text = Some(g.clone());
9393 self.state.event(
9394 "github-text",
9395 format!("asking the owner about the withheld text ({})", q.short()),
9396 );
9397 self.state.save()?;
9398 store
9399 .put_executable(&mut q)
9400 .context("file the posting-gate question")?;
9401 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9402 tracing::warn!(
9403 "could not notify about posting-gate question {}: {e:#}",
9404 q.short()
9405 );
9406 }
9407 self.park_on_github_text(&g)?;
9408 return Ok(None);
9409 }
9410 }
9411 let resolved = g.chosen_title.clone();
9412 let original = g.choice == Some(GithubTextChoice::Original);
9413 self.state.event(
9415 "github-text",
9416 if original {
9417 format!(
9418 "posting the original text for: {}",
9419 match (g.title, g.body) {
9420 (true, true) => "title, description",
9421 (true, false) => "title",
9422 _ => "description",
9423 }
9424 )
9425 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9426 "posting the saved rewrite title and description".to_owned()
9427 } else if resolved.is_some() {
9428 "posting the owner's replacement title".to_owned()
9429 } else {
9430 "posting the neutral text for the withheld fields".to_owned()
9431 },
9432 );
9433 self.state.github_text = Some(g.clone());
9434 self.state.save()?;
9435 if g.choice == Some(GithubTextChoice::Rewrite)
9436 && let Some(r) = g.rewrite.as_ref()
9437 {
9438 return Ok(Some(PrMessage {
9439 title: r.title.clone(),
9440 body: r.body.clone(),
9441 }));
9442 }
9443 let w = gt::Withheld {
9444 title: g.title,
9445 body: g.body,
9446 categories: g.categories.clone(),
9447 };
9448 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9449 if let Some(t) = resolved {
9450 title = t;
9451 }
9452 let english = gt::LanguageDecision {
9454 title_english: true,
9455 body_english: true,
9456 body_complete: true,
9457 };
9458 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9459 if original {
9460 if g.title {
9463 title = pr.title.clone();
9464 }
9465 if g.body {
9466 body = pr.body.clone();
9467 }
9468 }
9469 Ok(Some(PrMessage { title, body }))
9470 }
9471
9472 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9473 self.state.github_text = Some(g.clone());
9474 self.state.end_pass_parked(ParkCause::Question);
9475 self.state.parked = true;
9476 self.state.event(
9477 "github-text",
9478 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9479 );
9480 self.state.save()
9481 }
9482
9483 async fn judge_pr_language(
9486 &mut self,
9487 cwd: &Path,
9488 pr: &PrMessage,
9489 ) -> Option<crate::github_text::LanguageDecision> {
9490 self.state.config.roles.language_judge.as_ref()?;
9491 if !self.state.config.graph.language_gate() {
9492 return None;
9493 }
9494 let decision =
9495 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9496 self.state.event(
9497 "github-text",
9498 if decision.is_some() {
9499 "language decided by the language judge"
9500 } else {
9501 "language judge unavailable; using built-in heuristics"
9502 },
9503 );
9504 decision
9505 }
9506
9507 fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9508 Box::pin(self.merge_inner())
9509 }
9510
9511 async fn merge_inner(&mut self) -> Result<()> {
9512 if self.resource_halt()? || self.hold_base_sync().await? {
9513 return Ok(());
9514 }
9515 if self
9530 .state
9531 .base_sync
9532 .as_ref()
9533 .is_some_and(|s| s.conflict.is_some())
9534 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9535 != Some(RunStatus::Gating)
9536 || !self.state.gate_allows_progress()
9544 {
9545 return Ok(());
9546 }
9547 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9556 return Ok(());
9557 }
9558 let Some(winner) = self.state.winner().cloned() else {
9559 return Ok(());
9560 };
9561 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9562 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9563 self.state.status = RunStatus::Blocked;
9564 self.state.event(
9565 "gate",
9566 "winner changed after gate; fresh verification required",
9567 );
9568 self.state.save()?;
9569 return Ok(());
9570 }
9571 let wait = self.repository_wait();
9572 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9573 return Ok(());
9574 }
9575 let repo = self.state.repo.clone();
9576 let base = self.state.base_branch.clone();
9577 let mode = self.state.config.merge.mode;
9578 let style = self.state.config.merge.style;
9579 let facts = if is_review_run(&self.state) {
9580 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9581 let start = review_base(
9582 &repo,
9583 &self.state.config.merge.remote,
9584 &base,
9585 &self.state.base_commit,
9586 &winner.branch,
9587 )
9588 .await;
9589 branch_facts(&repo, &start, &winner.branch).await
9590 } else {
9591 None
9592 };
9593 let leaked = if is_review_run(&self.state) {
9596 leaked_subjects(&self.state, &winner.branch).await
9597 } else {
9598 Some(Vec::new())
9599 };
9600 let Some(pr) = self
9604 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9605 .await?
9606 else {
9607 return Ok(());
9608 };
9609 let message = pr.commit_message();
9610
9611 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9612 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9613 return Ok(());
9614 }
9615 let outcome = match mode {
9616 MergeMode::None => MergeOutcome {
9617 mode,
9618 ok: true,
9619 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9620 empty: false,
9621 },
9622 MergeMode::Pr | MergeMode::Local
9623 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9624 {
9625 MergeOutcome {
9626 mode,
9627 ok: false,
9628 detail: empty_candidate_detail(&self.state, &base),
9629 empty: true,
9630 }
9631 }
9632 MergeMode::Local => {
9633 let on = git::current_branch(&repo).await?;
9634 if on.as_deref() != Some(base.as_str()) {
9635 MergeOutcome {
9636 mode,
9637 ok: false,
9638 detail: format!(
9639 "{} has {} checked out, not the base branch {base}",
9640 repo.display(),
9641 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9642 ),
9643 empty: false,
9644 }
9645 } else if !git::is_clean(&repo).await? {
9646 MergeOutcome {
9647 mode,
9648 ok: false,
9649 detail: format!("{} is dirty; refusing to merge", repo.display()),
9650 empty: false,
9651 }
9652 } else {
9653 let out = match style {
9654 MergeStyle::Merge => {
9655 git::merge_no_ff(&repo, &winner.branch, &message).await?
9656 }
9657 MergeStyle::Squash => {
9658 git::merge_squash(&repo, &winner.branch, &message).await?
9659 }
9660 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9661 };
9662 MergeOutcome {
9663 mode,
9664 ok: out.ok(),
9665 detail: if out.ok() { out.stdout } else { out.stderr },
9666 empty: false,
9667 }
9668 }
9669 }
9670 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9671 self.state.pr_publication_pending = true;
9672 self.state.save()?;
9673 let out =
9674 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9675 self.state.pr_publication_pending = out.is_err();
9676 MergeOutcome {
9677 mode,
9678 ok: out.is_ok(),
9679 detail: match out {
9680 Ok(url) => url,
9681 Err(e) => format!("{e:#}"),
9682 },
9683 empty: false,
9684 }
9685 }
9686 MergeMode::Pr => {
9687 let remote = self.state.config.merge.remote.clone();
9688 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9689 let pushed = if self.state.foreign_checkout.is_some() {
9690 let refspec = format!("{delivery_head}:refs/heads/{public}");
9691 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9692 } else {
9693 git::push(&winner.worktree, &remote, &winner.branch).await?
9694 };
9695 if !pushed.ok() {
9696 MergeOutcome {
9697 mode,
9698 ok: false,
9699 detail: pushed.stderr,
9700 empty: false,
9701 }
9702 } else {
9703 if self.state.foreign_checkout.is_some() {
9704 self.state.pr_pushed_head = Some(delivery_head.clone());
9705 }
9706 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9712 let out = match pr_merge_plan(found) {
9713 PrPlan::Create => {
9714 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9715 .await
9716 }
9717 PrPlan::Adopt { url, title } => {
9718 self.state
9719 .event("merge", format!("Pr: adopted open pull request {url}"));
9720 if self.state.explicit_github_text_selection() {
9721 land::set_pr_text_verbatim(
9724 &winner.worktree,
9725 &url,
9726 &pr.title,
9727 &pr.body,
9728 )
9729 .await
9730 .map(|()| url)
9731 } else {
9732 if title != pr.title
9733 && (!is_review_run(&self.state)
9734 || leaked
9735 .as_deref()
9736 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9737 && let Err(e) =
9738 if self.state.github_text.as_ref().is_some_and(|g| {
9739 g.resolved
9740 && g.title
9741 && g.choice == Some(GithubTextChoice::Original)
9742 }) {
9743 land::set_pr_title_verbatim(
9745 &winner.worktree,
9746 &url,
9747 &pr.title,
9748 )
9749 .await
9750 } else {
9751 land::set_pr_title(
9752 &mut self.state,
9753 &winner.worktree,
9754 &url,
9755 &pr.title,
9756 )
9757 .await
9758 }
9759 {
9760 tracing::warn!("could not refresh title of {url}: {e:#}");
9761 self.state
9762 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9763 }
9764 Ok(url)
9765 }
9766 }
9767 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9768 };
9769 match out {
9770 Ok(url) => MergeOutcome {
9771 mode,
9772 ok: true,
9773 detail: url,
9774 empty: false,
9775 },
9776 Err(e) => MergeOutcome {
9777 mode,
9778 ok: false,
9779 detail: e.to_string(),
9780 empty: false,
9781 },
9782 }
9783 }
9784 }
9785 };
9786
9787 self.state.status = match (mode, outcome.ok) {
9788 (MergeMode::None, _) => RunStatus::Ready,
9789 (_, true) if outcome.empty => RunStatus::Ready,
9790 (MergeMode::Pr, true) => RunStatus::Ready,
9791 (_, true) => RunStatus::Merged,
9792 (_, false) => RunStatus::Blocked,
9793 };
9794 self.state.event(
9795 "merge",
9796 format!(
9797 "{:?}: {}",
9798 mode,
9799 outcome.detail.lines().next().unwrap_or("")
9800 ),
9801 );
9802 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9803 self.state.record_github_text_publication(&outcome.detail);
9804 }
9805 self.state.merge = Some(outcome);
9806 self.state.save()?;
9807
9808 if self.state.config.graph.land
9814 && mode == MergeMode::Pr
9815 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9816 {
9817 self.run_land().await?;
9818 }
9819 self.settle_questions();
9824 Ok(())
9825 }
9826
9827 fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9838 Box::pin(self.run_land_inner())
9839 }
9840
9841 async fn run_land_inner(&mut self) -> Result<()> {
9842 crate::pr_head::ensure_owner(&self.state)?;
9843 let url = self
9844 .state
9845 .pr
9846 .as_ref()
9847 .map(|pr| pr.url.clone())
9848 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9849 .unwrap_or_default();
9850 let url = url.lines().next().unwrap_or("").trim().to_owned();
9851 if !url.starts_with("http") {
9852 return Ok(());
9853 }
9854 loop {
9857 match land::land(&mut self.state, &url).await {
9858 Ok(_)
9859 if self
9860 .state
9861 .merge_repairs
9862 .last()
9863 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9864 {
9865 if self.repair_merge_review().await? {
9866 continue;
9867 }
9868 }
9869 Ok(_) if self.state.resource_block.is_some() => {
9870 return self.stop_for_resource();
9871 }
9872 Ok(pr) if self.state.parked => {
9873 let _ = pr;
9877 }
9878 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9879 self.state.status = RunStatus::Superseded;
9880 self.state.save()?;
9881 }
9882 Ok(pr) => {
9883 self.state.status = match pr.state {
9884 land::PrLifecycle::Merged => RunStatus::Merged,
9885 _ => RunStatus::Blocked,
9886 };
9887 if bump::should_release_bump(self.state.status)
9894 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9895 {
9896 self.state
9902 .event("bump", format!("release bump skipped: {e:#}"));
9903 }
9904 if self.state.status == RunStatus::Merged {
9907 crate::followup::after_merge(&mut self.state, &pr.url).await;
9908 }
9909 self.state.save()?;
9910 }
9911 Err(e) => {
9912 if e.is::<crate::repo_lock::ResourceBlocked>() {
9913 return Err(e);
9914 }
9915 self.state.status = RunStatus::Blocked;
9916 self.state.event("land", format!("gave up: {e}"));
9917 self.state.save()?;
9918 }
9919 }
9920 break;
9921 }
9922 Ok(())
9923 }
9924
9925 async fn repair_merge_review(&mut self) -> Result<bool> {
9928 use crate::run::MergeRepairPhase as Phase;
9929 crate::pr_head::ensure_owner(&self.state)?;
9930 let i = self.state.merge_repairs.len() - 1;
9931 let request = self.state.merge_repairs[i].clone();
9932 if request.phase == Phase::Fixing {
9933 let restored = match self.state.winner().cloned() {
9934 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9935 None => Err(anyhow::anyhow!("repair has no winning branch")),
9936 };
9937 let why = match restored {
9938 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9939 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9940 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9941 };
9942 self.hold_merge_repair(i, why)?;
9943 return Ok(false);
9944 }
9945 if request.phase == Phase::Reviewing && self.state.gate_ran {
9950 let winner = self
9951 .state
9952 .winner()
9953 .context("repair has no winning branch")?;
9954 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9955 if self.state.gate_head.as_deref() != Some(head.as_str())
9956 || !git::is_clean(&winner.worktree).await?
9957 {
9958 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9959 return Ok(false);
9960 }
9961 }
9962 let failure = if i >= self.state.config.graph.land_rounds
9963 || self.state.config.graph.review_rounds == 0
9964 {
9965 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9966 } else {
9967 None
9968 };
9969 if let Some(why) = failure {
9970 self.hold_merge_repair(i, why)?;
9971 return Ok(false);
9972 }
9973 if request.phase == Phase::Requested {
9974 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9975 Ok(pr) => pr,
9976 Err(e) => {
9977 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9978 return Ok(false);
9979 }
9980 };
9981 self.state.merge_repairs[i].phase = Phase::Fixing;
9982 self.state.land_approval = None;
9983 self.state.land_armed_head = None;
9984 self.state.save()?;
9985 let evidence = serde_json::to_string_pretty(&request)?;
9988 let reason = format!(
9989 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9990 );
9991 let budget = self.state.config.graph.land_rounds;
9992 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9993 match result {
9994 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9995 Ok(land::Fixed::Deferred) => {
9996 let worktree = self
10000 .state
10001 .winner()
10002 .context("repair has no winning branch")?
10003 .worktree
10004 .clone();
10005 self.graft_restore(&worktree, &request.head).await?;
10006 self.state.merge_repairs[i].phase = Phase::Requested;
10007 self.stop_for_resource()?;
10008 return Ok(false);
10009 }
10010 Ok(land::Fixed::Failed(why)) => {
10011 self.hold_merge_repair(
10012 i,
10013 format!(
10014 "repair failed: {why}; inspect the branch and run an explicit review"
10015 ),
10016 )?;
10017 return Ok(false);
10018 }
10019 Err(e) => {
10020 self.hold_merge_repair(
10021 i,
10022 format!(
10023 "repair failed: {e:#}; inspect the branch and run an explicit review"
10024 ),
10025 )?;
10026 return Ok(false);
10027 }
10028 }
10029 self.begin_merge_repair_review(i)?;
10033 }
10034 self.review_loop().await?;
10035 self.gate().await?;
10036 if self.resource_halt()? || self.state.parked {
10037 return Ok(false);
10038 }
10039 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10040 != Some(RunStatus::Gating)
10041 || !self.state.gate_allows_progress()
10042 {
10043 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10044 return Ok(false);
10045 }
10046 let winner = self
10049 .state
10050 .winner()
10051 .cloned()
10052 .context("repair has no winning branch")?;
10053 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10054 if self.state.gate_head.as_deref() != Some(after.as_str())
10055 || !git::is_clean(&winner.worktree).await?
10056 {
10057 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10058 return Ok(false);
10059 }
10060 let expected = self
10061 .state
10062 .pr_pushed_head
10063 .as_deref()
10064 .unwrap_or(&request.head)
10065 .to_owned();
10066 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10067 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10070 return Ok(false);
10071 }
10072 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10073 let refspec = format!("{after}:refs/heads/{public}");
10074 let push = git::git_raw(
10075 &winner.worktree,
10076 &["push", &lease, &self.state.config.merge.remote, &refspec],
10077 )
10078 .await;
10079 match push {
10080 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10081 Ok(out) => {
10082 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10083 return Ok(false);
10084 }
10085 Err(e) => {
10086 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10087 return Ok(false);
10088 }
10089 }
10090 self.state.merge_repairs[i].phase = Phase::Complete;
10091 self.state.status = RunStatus::Landing;
10092 self.state.event(
10093 "repair",
10094 "new review and gate completed; fresh owner merge decision required",
10095 );
10096 self.state.save()?;
10097 Ok(true)
10098 }
10099
10100 fn merge_repair_artifacts(&self) -> PathBuf {
10101 let artifacts = agent::artifacts_dir(&self.state.dir());
10102 match self
10103 .state
10104 .merge_repairs
10105 .last()
10106 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10107 {
10108 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10109 None => artifacts,
10110 }
10111 }
10112
10113 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10114 if let Some(e2e) = self.state.head_e2e.take() {
10115 self.state
10116 .verification_history
10117 .push(crate::run::VerificationEvidence {
10118 superseded_by: None,
10119 archived_at: Some(Timestamp::now()),
10120 fixes_spent: Some(self.state.gate_fixes.len()),
10121 gate_head: None,
10122 gate_ran: false,
10123 gate: Vec::new(),
10124 e2e: Some(e2e),
10125 });
10126 }
10127 self.state.reviews.clear();
10128 self.state.gate.clear();
10129 self.state.gate_ran = false;
10130 self.state.gate_head = None;
10131 self.state.gate_fixes.clear();
10132 self.state.gate_repair_e2e = None;
10133 self.state.pre_gate.clear();
10134 self.state.pre_gate_commit = None;
10135 self.state.contested_handoff = None;
10136 self.state.land_reruns = None;
10137 self.state.land_approval = None;
10138 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10139 self.state.status = RunStatus::Reviewing;
10140 self.state.save()
10141 }
10142
10143 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10144 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10145 self.state.merge_repairs[index].failure = Some(why.clone());
10146 self.state.status = RunStatus::Blocked;
10147 self.state.event("repair", why);
10148 self.state.save()
10149 }
10150
10151 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10155 if let Some(existing) = self.state.seats.get(key)
10156 && existing.agent == agent
10157 {
10158 return existing.clone();
10159 }
10160 let fresh = if self.state.seats.contains_key(key) {
10163 handover_seat(key, agent, self.state.next_seat_seed())
10164 } else {
10165 SeatState::new(key, agent, self.state.seed)
10166 };
10167 self.state.seats.insert(key.to_owned(), fresh.clone());
10168 fresh
10169 }
10170
10171 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10177 match self.state.seats.get(key) {
10178 Some(s) if s.agent != spec.id => {
10179 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10180 }
10181 _ => spec,
10182 }
10183 }
10184
10185 fn view(&self, c: &Candidate) -> CandidateView {
10187 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10188 .unwrap_or_default();
10189 let (patch, _) = blind::sanitize_patch(
10190 &format!("candidate {} patch", c.label),
10191 &raw,
10192 &self.state.config.blind,
10193 );
10194 CandidateView {
10195 label: c.label,
10196 branch: c.branch.clone(),
10197 summary: c.summary.clone(),
10198 stat: c.stat.clone(),
10199 patch,
10200 }
10201 }
10202
10203 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10205 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10206 prompt::judge(
10207 "(see above)",
10208 &views,
10209 self.roles.judges.len(),
10210 base_short,
10211 "en",
10212 )
10213 }
10214
10215 fn vote_prompt_full(
10221 &self,
10222 j: usize,
10223 labels: &[char],
10224 language: &str,
10225 candidates: &[Candidate],
10226 base_short: &str,
10227 ) -> String {
10228 let mut text = format!(
10229 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10230 prompt::final_vote(labels, language),
10231 self.state.instruction,
10232 self.candidate_block(candidates, base_short)
10233 );
10234 if let Some(own) = self
10235 .state
10236 .judgements
10237 .get(j)
10238 .filter(|r| !r.ranking.is_empty())
10239 {
10240 let reasons = own
10241 .reasons
10242 .iter()
10243 .map(|(k, v)| format!("- {k}: {v}"))
10244 .collect::<Vec<_>>()
10245 .join("\n");
10246 text.push_str(&format!(
10247 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10248 own.ranking.iter().collect::<String>(),
10249 if reasons.is_empty() {
10250 ""
10251 } else {
10252 ", because:\n"
10253 }
10254 ));
10255 }
10256 if !self.state.deliberation.is_empty() {
10257 text.push_str("\n# What was argued before this vote\n");
10258 for t in self.transcript(&[], j) {
10259 text.push_str(&format!(
10260 "\n## {}{}\n\n{}\n",
10261 t.who,
10262 if t.is_self { " (you)" } else { "" },
10263 t.body.trim()
10264 ));
10265 }
10266 }
10267 text
10268 }
10269
10270 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10277 let mut turns = Vec::new();
10278 for j in &self.state.judgements {
10279 if j.ranking.is_empty() {
10280 continue;
10281 }
10282 let reasons = j
10283 .reasons
10284 .iter()
10285 .map(|(k, v)| format!("- {k}: {v}"))
10286 .collect::<Vec<_>>()
10287 .join("\n");
10288 turns.push(Turn {
10289 who: format!("Judge {} (opening ranking)", j.judge),
10290 is_self: j.judge == self_idx + 1,
10291 body: format!(
10292 "Ranked {}{}{reasons}",
10293 j.ranking.iter().collect::<String>(),
10294 if reasons.is_empty() {
10295 ""
10296 } else {
10297 ", because:\n"
10298 }
10299 ),
10300 });
10301 }
10302 for t in self
10303 .state
10304 .deliberation
10305 .iter()
10306 .flat_map(|r| r.turns.iter())
10307 .chain(current)
10308 {
10309 turns.push(Turn {
10310 who: format!("Judge {}", t.judge),
10311 is_self: t.judge == self_idx + 1,
10312 body: t.body.clone(),
10313 });
10314 }
10315 turns
10316 }
10317}
10318
10319fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10321 agent::has_session(spec.kind, seat, sessions)
10322}
10323
10324fn next_untried_in_roster<'a>(
10345 roster: &'a [AgentSpec],
10346 start: usize,
10347 tried: &BTreeSet<String>,
10348) -> Option<&'a AgentSpec> {
10349 roster
10350 .get(start + 1..)?
10351 .iter()
10352 .find(|s| !tried.contains(&s.id))
10353}
10354
10355fn pick_successor<'a>(
10376 roster: &'a [AgentSpec],
10377 start: usize,
10378 tried: &BTreeSet<String>,
10379 carried: Option<&BTreeSet<String>>,
10380 others: &BTreeSet<String>,
10381) -> Option<&'a AgentSpec> {
10382 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10383 roster
10384 .get(start + 1..)
10385 .and_then(|tail| {
10386 tail.iter()
10387 .filter(free)
10388 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10389 })
10390 .or_else(|| {
10391 carried
10392 .is_some()
10393 .then(|| roster.iter().find(free))
10394 .flatten()
10395 })
10396 .or_else(|| match carried {
10397 Some(c) => next_for_seat(roster, start, tried, c),
10398 None => next_untried_in_roster(roster, start, tried),
10399 })
10400}
10401
10402fn next_for_seat<'a>(
10412 roster: &'a [AgentSpec],
10413 start: usize,
10414 round_tried: &BTreeSet<String>,
10415 carried_failed: &BTreeSet<String>,
10416) -> Option<&'a AgentSpec> {
10417 roster
10418 .get(start + 1..)?
10419 .iter()
10420 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10421 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10422}
10423
10424fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10430 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10431 return spec;
10432 };
10433 let ok = |id: &str| !h.failed.contains(id);
10434 if let Some(last) = h.last_ok.as_deref()
10435 && ok(last)
10436 && let Some(s) = roster.iter().find(|s| s.id == last)
10437 {
10438 return s.clone();
10439 }
10440 if ok(&spec.id) {
10441 return spec;
10442 }
10443 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10444 roster
10445 .get(from + 1..)
10446 .into_iter()
10447 .flatten()
10448 .chain(roster.iter())
10449 .find(|s| ok(&s.id))
10450 .cloned()
10451 .unwrap_or(spec)
10452}
10453
10454pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10458 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10459}
10460
10461const TIMED_OUT: &str = "timed out";
10464
10465impl FailClass {
10466 fn of(out: &AgentOutcome) -> Option<Self> {
10468 match out {
10469 AgentOutcome::Ok(_) => None,
10470 AgentOutcome::Quota(_) => Some(Self::Quota),
10471 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10472 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10473 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10474 AgentOutcome::ResourceBlocked(_) => None,
10477 }
10478 }
10479
10480 fn stem_word(&self) -> &'static str {
10482 match self {
10483 Self::Quota => "quota",
10484 _ => "handover",
10485 }
10486 }
10487}
10488
10489fn failure_signature(msg: &str) -> String {
10493 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10494 if let Some(code) = line
10497 .strip_prefix("process exited with no usable output (exit code ")
10498 .and_then(|s| s.strip_suffix(')'))
10499 .and_then(|s| s.parse::<i32>().ok())
10500 {
10501 line = format!("exited with some({code}) and no usable output");
10502 } else if matches!(
10503 line.as_str(),
10504 "process exited with no usable output (no exit code reported)"
10505 | "process exited with no usable output (exit code unavailable)"
10506 ) {
10507 line = "exited with none and no usable output".to_owned();
10508 }
10509 let mut out = Vec::new();
10510 for word in line.split_whitespace() {
10511 if word.contains('/') || word.contains('\\') {
10512 out.push("<path>".to_owned());
10513 continue;
10514 }
10515 let mut w = String::new();
10516 let mut in_digits = false;
10517 for c in word.chars() {
10518 if c.is_ascii_digit() {
10519 if !in_digits {
10520 w.push('#');
10521 }
10522 in_digits = true;
10523 } else {
10524 in_digits = false;
10525 w.push(c);
10526 }
10527 }
10528 out.push(w);
10529 }
10530 out.join(" ").chars().take(120).collect()
10531}
10532
10533fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10540 match cur {
10541 FailClass::Quota | FailClass::Timeout => true,
10542 FailClass::Other(_) => prev != Some(cur),
10543 }
10544}
10545
10546fn fail_reason(out: &AgentOutcome) -> String {
10548 match out {
10549 AgentOutcome::Ok(_) => String::new(),
10550 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10551 AgentOutcome::Dropped(o) => format!(
10552 "the CLI dropped the stream ({})",
10553 o.dropped
10554 .as_ref()
10555 .map(|d| d.why.as_str())
10556 .unwrap_or("it ended without delivering its answer")
10557 ),
10558 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10559 e.lines().next().unwrap_or("").chars().take(160).collect()
10560 }
10561 }
10562}
10563
10564pub(crate) fn record_handover(
10567 state: &mut RunState,
10568 node: &str,
10569 seat: &str,
10570 from: &str,
10571 to: &str,
10572 class: &FailClass,
10573 reason: &str,
10574) {
10575 let message = if *class == FailClass::Quota {
10576 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10577 } else {
10578 format!("{seat}: handed over {from} -> {to} ({reason})")
10579 };
10580 state.event(node, message);
10581 state.handovers.push(Handover {
10582 at: Timestamp::now(),
10583 node: node.to_owned(),
10584 seat: seat.to_owned(),
10585 from: from.to_owned(),
10586 to: to.to_owned(),
10587 reason: reason.to_owned(),
10588 });
10589}
10590
10591fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10605 commands.iter().any(|c| c.exit_code.is_none())
10606}
10607
10608fn verified_noop_claim(
10621 usable: bool,
10622 commands: &[agent::CommandEvidence],
10623 text: &str,
10624) -> Option<String> {
10625 (usable && !has_unconfirmed_command(commands))
10626 .then(|| verdict::verified_noop(text))
10627 .flatten()
10628}
10629
10630fn short(commit: &str) -> String {
10631 commit.chars().take(7).collect()
10632}
10633
10634fn make_executable(path: &Path) -> Result<()> {
10635 #[cfg(unix)]
10636 {
10637 use std::os::unix::fs::PermissionsExt as _;
10638 let mut perms = std::fs::metadata(path)?.permissions();
10639 perms.set_mode(0o755);
10640 std::fs::set_permissions(path, perms)?;
10641 }
10642 #[cfg(not(unix))]
10643 {
10644 let _ = path;
10645 }
10646 Ok(())
10647}
10648
10649struct WaveCtx<'a> {
10656 run: &'a str,
10659 node: &'a str,
10661 prompts: &'a Prompts,
10662 cache: Option<&'a Path>,
10664 round: Option<usize>,
10667 carry_seats: bool,
10671 pauses: Option<(&'a Pause, &'a Pause)>,
10674}
10675
10676fn repository_wait(
10678 state: &RunState,
10679 pauses: Option<(&Pause, &Pause)>,
10680) -> crate::repo_lock::WaitPolicy {
10681 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10682 let inherited = crate::repo_lock::WaitPolicy::current();
10683 crate::repo_lock::WaitPolicy::new(
10684 Duration::from_secs(state.config.graph.timeout_resource_wait),
10685 move || {
10686 inherited.cancelled()
10687 || pauses
10688 .as_ref()
10689 .is_some_and(|(a, b)| a.parked() || b.parked())
10690 },
10691 )
10692}
10693
10694fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10695 if error.is::<crate::repo_lock::ResourceBlocked>() {
10696 state.resource_block = Some(crate::run::ResourceBlock {
10697 cause: crate::run::ResourceBlockCause::Other,
10698 node: "repository".into(),
10699 kind: if error.to_string().contains("disk pressure") {
10700 "disk pressure"
10701 } else if error.to_string().contains("disk admission") {
10702 "disk admission"
10703 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10704 "incremental cache admission"
10705 } else {
10706 "repository coordination"
10707 }
10708 .into(),
10709 detail: error.to_string(),
10710 at: Timestamp::now(),
10711 });
10712 state.event("resource", error.to_string());
10713 }
10714}
10715
10716async fn coordinate_operation(
10717 state: &mut RunState,
10718 cwd: &Path,
10719 wait: &crate::repo_lock::WaitPolicy,
10720) -> Result<Option<crate::repo_lock::SeatLease>> {
10721 let result =
10722 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10723 .await;
10724 if let Err(e) = &result {
10725 record_repository_block(state, e);
10726 }
10727 result
10728}
10729
10730async fn run_one(
10732 job: SeatJob,
10733 sem: Arc<Semaphore>,
10734 ctx: &WaveCtx<'_>,
10735 state: &mut RunState,
10736 attempt: usize,
10737) -> (SeatState, AgentOutcome) {
10738 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10739 .await
10740 .pop()
10741 .expect("one job in, one result out");
10742 (seat, out)
10743}
10744
10745async fn wave(
10751 jobs: Vec<SeatJob>,
10752 sem: Arc<Semaphore>,
10753 ctx: &WaveCtx<'_>,
10754 state: &mut RunState,
10755 attempt: usize,
10756) -> Vec<(usize, SeatState, AgentOutcome)> {
10757 let WaveCtx {
10758 run,
10759 node,
10760 prompts,
10761 cache,
10762 round,
10763 carry_seats: _,
10764 pauses,
10765 } = *ctx;
10766 for job in &jobs {
10767 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10768 }
10769 if let Err(e) = state.save() {
10770 tracing::warn!("could not persist in-progress seats: {e:#}");
10775 }
10776 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10792 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10796 let cache_guard = if let Some(cache_dir) = cache {
10797 if jobs_had_a_writer {
10798 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10799 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10800 match acquire_cache_lease(
10801 state,
10802 cache_dir,
10803 &owner,
10804 resource_limit,
10805 node,
10806 node,
10807 &cancel,
10808 )
10809 .await
10810 {
10811 Ok(Some(g)) => Some(g),
10812 Ok(None) => {
10815 if let Some(block) = &mut state.resource_block {
10816 block.cause = crate::run::ResourceBlockCause::Cancelled {
10817 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10818 };
10819 }
10820 return block_wave(
10821 jobs,
10822 state,
10823 node,
10824 attempt,
10825 "parked while waiting for the shared build cache",
10826 );
10827 }
10828 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10831 Err(e) => {
10832 let why = format!("{e:#}");
10836 state.event(
10837 "resource",
10838 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10839 );
10840 return block_wave(jobs, state, node, attempt, &why);
10841 }
10842 }
10843 } else {
10844 None
10845 }
10846 } else {
10847 None
10848 };
10849 state.seats_spawning(node, attempt);
10852 if let Err(e) = state.save() {
10853 tracing::warn!("could not persist the start of a wave: {e:#}");
10854 }
10855 let slot_limit = resource_limit
10859 + jobs
10860 .iter()
10861 .map(|j| j.timeout)
10862 .max()
10863 .unwrap_or_default()
10864 .saturating_mul(jobs.len() as u32);
10865 let mut set = tokio::task::JoinSet::new();
10866 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10869 let overlay = prompts.overlay(node);
10870 let repository_wait = repository_wait(state, pauses);
10871 for (i, mut job) in jobs.into_iter().enumerate() {
10872 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10873 if cache.is_some() {
10874 job.prompt.push('\n');
10875 job.prompt
10876 .push_str(&prompt::build_cache_note(node, job.allow_write));
10877 }
10878 let sem = Arc::clone(&sem);
10879 let run = run.to_owned();
10880 let node = node.to_owned();
10881 let attachments = if node == "implement" {
10884 state.attachments.clone()
10885 } else {
10886 Vec::new()
10887 };
10888 let cache = cache
10899 .filter(|_| job.allow_write && cache_guard.is_some())
10900 .map(Path::to_path_buf);
10901 let spawned_tx = spawned_tx.clone();
10902 let repository_wait = repository_wait.clone();
10903 let coordination = state.config.coordination.clone();
10904 let fetched_base = state.fetched_base.clone();
10905 set.spawn(async move {
10906 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10910 Ok(p) => p,
10911 Err(_) => {
10912 return (
10913 i,
10914 job.seat,
10915 AgentOutcome::ResourceBlocked(format!(
10916 "no parallel slot became free within {}s",
10917 slot_limit.as_secs()
10918 )),
10919 false,
10920 );
10921 }
10922 };
10923 spawned_tx.send(job.seat.key.clone()).ok();
10924 let mut seat = job.seat;
10925 let out = repository_wait
10926 .scope(agent::invoke(
10927 &job.spec,
10928 &mut seat,
10929 &Invocation {
10930 coordination: coordination.as_ref(),
10931 fetched_base: fetched_base.as_ref(),
10932 cwd: &job.cwd,
10933 prompt: &job.prompt,
10934 timeout: job.timeout,
10935 allow_write: job.allow_write,
10936 sessions: job.sessions,
10937 artifacts: &job.artifacts,
10938 stem: &job.stem,
10939 run: &run,
10940 node: &node,
10941 cache_dir: cache.as_deref(),
10942 attachments: &attachments,
10943 writable: &[],
10944 },
10945 ))
10946 .await;
10947 let interrupted = out
10948 .as_ref()
10949 .err()
10950 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10951 let out = match out {
10952 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10953 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10954 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10962 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10963 Ok(o) => AgentOutcome::Failed(format!(
10964 "process exited with no usable output ({})",
10965 crate::notices::exit_status(o.exit_code)
10966 )),
10967 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10968 AgentOutcome::ResourceBlocked(e.to_string())
10969 }
10970 Err(e) => AgentOutcome::Failed(e.to_string()),
10971 };
10972 (i, seat, out, interrupted)
10973 });
10974 }
10975 drop(spawned_tx);
10976 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10977 loop {
10978 let joined = tokio::select! {
10979 Some(key) = spawned_rx.recv() => {
10980 state.seat_spawned(&key);
10982 if let Err(e) = state.save() {
10983 tracing::warn!("could not persist a seat's start: {e:#}");
10984 }
10985 continue;
10986 }
10987 joined = set.join_next() => joined,
10988 };
10989 let Some(joined) = joined else {
10990 break;
10991 };
10992 let (i, seat, out, interrupted) = match joined {
10993 Ok(v) => v,
10994 Err(e) => {
10998 tracing::error!("agent task panicked: {e}");
10999 continue;
11000 }
11001 };
11002 state.seat_finished(&seat.key);
11003 if interrupted
11004 && node == "implement"
11005 && let Some(label) = state
11006 .candidates
11007 .iter()
11008 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11009 .map(|candidate| candidate.label)
11010 {
11011 state.candidate_preservation.insert(
11012 label,
11013 "interrupted implementation awaiting persistence".into(),
11014 );
11015 state.interrupted_implementations.insert(label);
11016 }
11017 if let AgentOutcome::ResourceBlocked(why) = &out
11018 && state.resource_block.is_none()
11019 {
11020 state.resource_block = Some(crate::run::ResourceBlock {
11021 cause: crate::run::ResourceBlockCause::Other,
11022 node: node.to_owned(),
11023 kind: if why.contains("disk pressure") {
11024 "disk pressure"
11025 } else if why.contains("disk admission") {
11026 "disk admission"
11027 } else if why.contains(crate::incremental::BLOCKED) {
11028 "incremental cache admission"
11029 } else if why.contains("repository coordination") {
11030 "repository coordination"
11031 } else {
11032 "parallel slot"
11033 }
11034 .to_owned(),
11035 detail: why.clone(),
11036 at: Timestamp::now(),
11037 });
11038 let activity = if interrupted {
11039 "agent execution interrupted"
11040 } else {
11041 "no agent was started"
11042 };
11043 state.event("resource", format!("{node}: {activity} - {why}"));
11044 }
11045 record_jobs(state, node, round, &seat.key, &out);
11046 if let Err(e) = state.save() {
11047 tracing::warn!("could not persist a seat's completion: {e:#}");
11048 }
11049 if collected.len() <= i {
11050 collected.resize_with(i + 1, || None);
11051 }
11052 collected[i] = Some((i, seat, out));
11053 }
11054 if state
11060 .active
11061 .values()
11062 .any(|a| a.node == node && a.attempt == attempt)
11063 {
11064 state
11065 .active
11066 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11067 if let Err(e) = state.save() {
11068 tracing::warn!("could not persist the end of a wave: {e:#}");
11069 }
11070 }
11071 if let Some(cache_dir) = cache
11078 && jobs_had_a_writer
11079 {
11080 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11081 }
11082 if let Some(guard) = cache_guard {
11083 guard.release();
11084 }
11085 collected.into_iter().flatten().collect()
11086}
11087
11088fn record_jobs(
11099 state: &mut RunState,
11100 node: &str,
11101 round: Option<usize>,
11102 seat: &str,
11103 out: &AgentOutcome,
11104) {
11105 let commands: &[agent::CommandEvidence] = match out {
11106 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11107 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11108 };
11109 let checked_at = Timestamp::now();
11110 for c in commands {
11111 state.jobs.push(JobRecord {
11112 node: node.to_owned(),
11113 round,
11114 seat: seat.to_owned(),
11115 id: c.id.clone(),
11116 description: c.description.clone(),
11117 checked_at,
11118 status: match c.exit_code {
11119 Some(0) => JobStatus::Completed,
11120 Some(_) => JobStatus::Failed,
11121 None => JobStatus::Unknown,
11122 },
11123 exit_code: c.exit_code,
11124 result_summary: c.result_summary.clone(),
11125 source: c.source.clone(),
11126 });
11127 }
11128}
11129
11130fn round_is_clean(
11151 blocking: usize,
11152 e2e_ok: bool,
11153 answered: usize,
11154 expected: usize,
11155 quota_missing: usize,
11156 policy: IncompleteReviewPolicy,
11157) -> bool {
11158 if blocking != 0 || !e2e_ok {
11159 return false;
11160 }
11161 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11162 return true;
11163 }
11164 answered > 0 && expected - answered <= quota_missing
11165}
11166
11167pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11191 if max_rounds == 0
11192 || reviews.last().is_some_and(|r| {
11193 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11194 })
11195 {
11196 return Some(RunStatus::Gating);
11197 }
11198 let last = reviews.last()?;
11199 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11200 if reviews.len() < max_rounds && !stagnant {
11201 return None;
11202 }
11203 if last.incomplete() && last.blocking == 0 {
11204 return Some(RunStatus::Blocked);
11205 }
11206 if last.e2e_status() == E2eStatus::ResourceBlocked
11207 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11208 {
11209 return None;
11210 }
11211 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11212 RunStatus::Gating
11213 } else {
11214 RunStatus::Blocked
11215 })
11216}
11217
11218pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11233 if nudged {
11234 (full / 4).max(Duration::from_secs(120)).min(full)
11235 } else {
11236 full
11237 }
11238}
11239
11240#[allow(clippy::too_many_arguments)]
11269async fn ask_json_wave<T>(
11270 jobs: Vec<SeatJob>,
11271 sem: Arc<Semaphore>,
11272 retries: usize,
11273 roster: &[AgentSpec],
11274 ctx: &WaveCtx<'_>,
11275 losses: &mut Vec<QuotaLoss>,
11276 state: &mut RunState,
11277 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11278) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11279where
11280 T: serde::de::DeserializeOwned + Send + 'static,
11281{
11282 ask_wave_with(
11283 jobs,
11284 sem,
11285 retries,
11286 roster,
11287 ctx,
11288 losses,
11289 state,
11290 &|text: &str| {
11291 let v = verdict::extract_json::<T>(text)?;
11292 validate(&v)?;
11293 Ok(v)
11294 },
11295 )
11296 .await
11297}
11298
11299#[allow(clippy::too_many_arguments)]
11308async fn ask_wave_with<T>(
11309 jobs: Vec<SeatJob>,
11310 sem: Arc<Semaphore>,
11311 retries: usize,
11312 roster: &[AgentSpec],
11313 ctx: &WaveCtx<'_>,
11314 losses: &mut Vec<QuotaLoss>,
11315 state: &mut RunState,
11316 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11317) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11318where
11319 T: Send + 'static,
11320{
11321 let n = jobs.len();
11322 let originals: Vec<SeatJob> = jobs;
11323 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11324 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11325 let mut nudges: Vec<usize> = vec![0; n];
11332 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11336 let mut tried: Vec<BTreeSet<String>> = specs
11337 .iter()
11338 .map(|s| BTreeSet::from([s.id.clone()]))
11339 .collect();
11340 let starts: Vec<usize> = specs
11341 .iter()
11342 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11343 .collect();
11344 let carry = ctx.carry_seats && !roster.is_empty();
11345 let carried: Vec<BTreeSet<String>> = originals
11348 .iter()
11349 .map(|j| {
11350 state
11351 .seat_history
11352 .get(&j.seat.key)
11353 .filter(|_| carry)
11354 .map(|h| h.failed.clone())
11355 .unwrap_or_default()
11356 })
11357 .collect();
11358 let mut prev: Vec<Option<FailClass>> = originals
11359 .iter()
11360 .map(|j| {
11361 state
11362 .seat_history
11363 .get(&j.seat.key)
11364 .filter(|_| carry)
11365 .and_then(|h| h.last_fail.clone())
11366 })
11367 .collect();
11368 let next_agent =
11369 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11370 let others: BTreeSet<String> = specs
11371 .iter()
11372 .enumerate()
11373 .filter(|(j, _)| *j != i)
11374 .map(|(_, s)| s.id.clone())
11375 .collect();
11376 pick_successor(
11377 roster,
11378 starts[i],
11379 tried,
11380 carry.then(|| &carried[i]),
11381 &others,
11382 )
11383 .cloned()
11384 };
11385 let mut fresh: Vec<Option<String>> = vec![None; n];
11386 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11387 let mut pending: Vec<usize> = (0..n).collect();
11388
11389 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11394 for round in 0..max_rounds {
11395 if pending.is_empty() {
11396 break;
11397 }
11398 let mut batch = Vec::with_capacity(pending.len());
11399 let mut renudged: Vec<&str> = Vec::new();
11400 for &i in &pending {
11401 let src = &originals[i];
11402 let full: &str = match &src.handover {
11407 Some(h) if specs[i].id != src.spec.id => h,
11408 _ => &src.prompt,
11409 };
11410 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11413 (
11414 full.to_owned(),
11415 src.timeout,
11416 format!("{}-{word}-{}", src.stem, specs[i].id),
11417 )
11418 } else if nudges[i] == 0 {
11419 (full.to_owned(), src.timeout, src.stem.clone())
11420 } else {
11421 renudged.push(src.seat.key.as_str());
11422 let why = done[i]
11423 .as_ref()
11424 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11425 .unwrap_or_else(|| "no parsable answer".to_owned());
11426 let nudge = prompt::nudge(&why);
11427 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11428 let prompt = if nudged {
11429 nudge
11430 } else {
11431 format!("{full}\n\n---\n\n{nudge}")
11432 };
11433 (
11434 prompt,
11435 retry_budget(src.timeout, nudged),
11436 format!("{}-retry{}", src.stem, nudges[i]),
11437 )
11438 };
11439 batch.push(SeatJob {
11440 spec: specs[i].clone(),
11441 seat: seats[i].clone(),
11442 cwd: src.cwd.clone(),
11443 prompt,
11444 timeout,
11445 allow_write: src.allow_write,
11446 sessions: src.sessions,
11447 artifacts: src.artifacts.clone(),
11448 stem,
11449 handover: None,
11450 });
11451 }
11452
11453 if !renudged.is_empty() {
11454 state.event(
11455 ctx.node,
11456 format!("retry {round}: re-asking {}", renudged.join(", ")),
11457 );
11458 }
11459 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11460 let mut still = Vec::new();
11461 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11462 seats[i] = seat;
11463 if let AgentOutcome::ResourceBlocked(why) = &out {
11466 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11467 continue;
11468 }
11469 let class = FailClass::of(&out);
11470 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11473 && nudges[i] < retries
11474 && !roster.is_empty();
11475 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11476 let next = should_hand_over(prev[i].as_ref(), &cur)
11477 .then(|| next_agent(i, &tried[i], &specs))
11478 .flatten();
11479 if carry {
11480 let h = state
11481 .seat_history
11482 .entry(originals[i].seat.key.clone())
11483 .or_default();
11484 h.failed.insert(specs[i].id.clone());
11485 h.last_fail = Some(cur.clone());
11486 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11487 h.last_ok = None;
11488 }
11489 if let Err(e) = state.save() {
11492 tracing::warn!("could not persist a seat's failure history: {e:#}");
11493 }
11494 }
11495 if let Some(next) = next {
11496 record_handover(
11497 state,
11498 ctx.node,
11499 &originals[i].seat.key,
11500 &specs[i].id,
11501 &next.id,
11502 &cur,
11503 &fail_reason(&out),
11504 );
11505 tried[i].insert(next.id.clone());
11506 prev[i] = Some(cur.clone());
11507 seats[i] =
11508 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11509 specs[i] = next;
11510 fresh[i] = Some(cur.stem_word().to_owned());
11511 nudges[i] = 0;
11512 done[i] = Some(Err(anyhow::anyhow!(
11513 "handed over after: {}",
11514 fail_reason(&out)
11515 )));
11516 still.push(i);
11517 continue;
11518 }
11519 }
11520 if carry
11521 && !nudge_first
11522 && let Some(cur) = class.clone()
11523 {
11524 let h = state
11526 .seat_history
11527 .entry(originals[i].seat.key.clone())
11528 .or_default();
11529 h.failed.insert(specs[i].id.clone());
11530 h.last_fail = Some(cur);
11531 }
11532 let parsed = match out {
11533 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11534 AgentOutcome::Quota(o) => {
11535 last_quota[i] = o.quota.clone();
11536 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11537 }
11538 AgentOutcome::Dropped(o) => {
11542 let why = o
11543 .dropped
11544 .as_ref()
11545 .map(|d| d.why.as_str())
11546 .unwrap_or("the CLI ended the stream without delivering its answer");
11547 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11548 }
11549 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11550 Err(anyhow::anyhow!(e))
11551 }
11552 };
11553 let quota = class == Some(FailClass::Quota);
11554 let failed = parsed.is_err();
11555 if !failed && matches!(ctx.node, "judge" | "vote") {
11556 state
11559 .quota
11560 .retain(|loss| loss.seat != originals[i].seat.key);
11561 }
11562 done[i] = Some(parsed);
11563 if carry && !failed {
11564 let h = state
11565 .seat_history
11566 .entry(originals[i].seat.key.clone())
11567 .or_default();
11568 h.failed.remove(&specs[i].id);
11569 h.last_ok = Some(specs[i].id.clone());
11570 h.last_fail = None;
11571 }
11572 let agent_failure = class.is_some()
11580 && !nudge_first
11581 && !roster.is_empty()
11582 && next_agent(i, &tried[i], &specs).is_some();
11583 if failed && !quota && !agent_failure && nudges[i] < retries {
11584 nudges[i] += 1;
11585 still.push(i);
11586 }
11587 }
11588 pending = still;
11589 }
11590
11591 for (i, q) in last_quota.into_iter().enumerate() {
11594 if let Some(quota) = q {
11595 losses.push(QuotaLoss::from_report(
11596 originals[i].seat.key.clone(),
11597 ctx.node,
11598 Some("a),
11599 ));
11600 }
11601 }
11602
11603 if state.status != RunStatus::Stalled
11606 && ctx.node != "deliberate"
11607 && n > 0
11608 && losses.len() == n
11609 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11610 {
11611 block_machine_outage(state, ctx.node, losses);
11612 state.quota.append(losses);
11613 }
11614
11615 seats
11616 .into_iter()
11617 .zip(done)
11618 .zip(nudges)
11619 .map(|((seat, res), attempts)| {
11620 (
11621 seat,
11622 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11623 attempts,
11624 )
11625 })
11626 .collect()
11627}
11628
11629pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11632 let mut auth: Vec<_> = losses
11633 .iter()
11634 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11635 .map(|loss| loss.agent.clone())
11636 .collect();
11637 auth.sort();
11638 auth.dedup();
11639 let (kind, detail) = if auth.is_empty() {
11640 (
11641 "agent usage limit",
11642 "All seats reached their agent usage limits; resume after usage is available"
11643 .to_owned(),
11644 )
11645 } else {
11646 (
11647 "agent authentication",
11648 format!(
11649 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11650 auth.join(", ")
11651 ),
11652 )
11653 };
11654 state.resource_block = Some(crate::run::ResourceBlock {
11655 node: node.to_owned(),
11656 kind: kind.to_owned(),
11657 cause: crate::run::ResourceBlockCause::Other,
11658 detail: detail.clone(),
11659 at: Timestamp::now(),
11660 });
11661 state.event(node, detail);
11662}
11663
11664fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11666 results
11667 .iter()
11668 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11669}
11670
11671fn block_wave(
11675 jobs: Vec<SeatJob>,
11676 state: &mut RunState,
11677 node: &str,
11678 attempt: usize,
11679 why: &str,
11680) -> Vec<(usize, SeatState, AgentOutcome)> {
11681 state.resource_wait = None;
11682 state
11683 .active
11684 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11685 if let Err(e) = state.save() {
11686 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11687 }
11688 jobs.into_iter()
11689 .enumerate()
11690 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11691 .collect()
11692}
11693
11694async fn acquire_cache_lease(
11707 state: &mut RunState,
11708 cache_dir: &Path,
11709 owner: &crate::cache::Owner,
11710 budget: Duration,
11711 node: &str,
11712 context: &str,
11713 cancel: &(dyn Fn() -> bool + Sync),
11714) -> Result<Option<crate::cache::Guard>> {
11715 let home = crate::run::home();
11716 let started = Instant::now();
11717 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11718 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11719 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11720 Err(e) => {
11721 state.event(
11722 "verify",
11723 format!("{context}: could not check the shared build cache: {e:#}"),
11724 );
11725 if let Err(e2) = state.save() {
11726 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11727 }
11728 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11729 }
11730 };
11731 state.event(
11732 "verify",
11733 format!(
11734 "{context}: waiting for the shared build cache at {} ({})",
11735 cache_dir.display(),
11736 busy.describe()
11737 ),
11738 );
11739 state.resource_wait = Some(crate::run::ResourceWait {
11742 node: context.to_owned(),
11743 kind: "shared build cache".to_owned(),
11744 holder: busy.describe(),
11745 since: Timestamp::now(),
11746 limit_secs: budget.as_secs(),
11747 });
11748 if let Err(e) = state.save() {
11749 tracing::warn!("could not persist a cache wait: {e:#}");
11750 }
11751 let remaining = budget.saturating_sub(started.elapsed());
11752 let waited = crate::cache::wait_for_unless(
11753 &home,
11754 cache_dir,
11755 owner,
11756 remaining,
11757 Duration::from_secs(5),
11758 cancel,
11759 )
11760 .await;
11761 state.resource_wait = None;
11762 let cause = match &waited {
11763 Ok(Some(_)) => None,
11764 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11765 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11766 Some(crate::run::ResourceBlockCause::Timeout {
11767 waited_secs: started.elapsed().as_secs(),
11768 limit_secs: budget.as_secs(),
11769 })
11770 }
11771 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11772 };
11773 if let Some(cause) = cause {
11774 let detail = match &waited {
11775 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11776 Err(error) => format!("{error:#}"),
11777 };
11778 state.resource_block = Some(crate::run::ResourceBlock {
11779 cause,
11780 node: node.to_owned(),
11781 kind: "shared build cache".into(),
11782 detail,
11783 at: Timestamp::now(),
11784 });
11785 state.event(
11786 "resource",
11787 state.resource_block.as_ref().unwrap().description(),
11788 );
11789 }
11790 if let Err(error) = state.save() {
11791 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11792 }
11793 waited
11794}
11795
11796#[derive(Debug)]
11801struct CacheCheckError(String);
11802
11803impl std::fmt::Display for CacheCheckError {
11804 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11805 f.write_str(&self.0)
11806 }
11807}
11808
11809impl std::error::Error for CacheCheckError {}
11810
11811fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11813 state
11814 .resource_block
11815 .get_or_insert_with(|| crate::run::ResourceBlock {
11816 cause: crate::run::ResourceBlockCause::Other,
11817 node: node.to_owned(),
11818 kind: "shared build cache".into(),
11819 detail: format!("{error:#}"),
11820 at: Timestamp::now(),
11821 });
11822 if let Err(error) = state.save() {
11823 tracing::warn!("could not persist cache resource block: {error:#}");
11824 }
11825}
11826
11827#[allow(clippy::too_many_arguments)]
11848async fn with_cache_lease<'s, F, Fut>(
11849 state: &'s mut RunState,
11850 cache_dir: Option<&Path>,
11851 node: &str,
11852 seat: &str,
11853 worktree: &Path,
11854 head: &str,
11855 budget: Duration,
11856 context: &str,
11857 wait: &crate::repo_lock::WaitPolicy,
11858 body: F,
11859) -> Result<(Vec<CommandOutcome>, bool)>
11860where
11861 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11862 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11863{
11864 let started = Instant::now();
11867 let home = crate::run::home();
11868 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11869 let guard = if let Some(cache_dir) = cache_dir {
11870 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11871 match acquire_cache_lease(
11872 state,
11873 cache_dir,
11874 &owner,
11875 resource_limit,
11876 node,
11877 context,
11878 &|| wait.cancelled(),
11879 )
11880 .await
11881 .and_then(|g| {
11882 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11883 }) {
11884 Ok(g) => Some(g),
11885 Err(e) => {
11886 record_cache_error(state, node, &e);
11887 return Ok((
11888 vec![CommandOutcome {
11889 command: "(waiting for the shared build cache)".into(),
11890 code: None,
11891 output_tail: e.to_string(),
11892 duration_ms: started.elapsed().as_millis() as u64,
11893 resource_blocked: true,
11894 }],
11895 false,
11896 ));
11897 }
11898 }
11899 } else {
11900 None
11901 };
11902 let _coordination = coordinate_operation(state, worktree, wait).await?;
11903 if let Some(cache_dir) = cache_dir {
11904 let identity = crate::cache::Identity::new(worktree, head);
11905 if let Err(e) = crate::cache::ensure_fresh(
11906 &home,
11907 cache_dir,
11908 &identity,
11909 &state.config.verify.cargo_manifests(),
11910 ) {
11911 record_cache_error(state, node, &e);
11912 state.event(
11913 "verify",
11914 format!(
11915 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11916 worktree.display(),
11917 short(head)
11918 ),
11919 );
11920 if let Some(guard) = guard {
11921 guard.release();
11922 }
11923 return Ok((
11924 vec![CommandOutcome {
11925 command: "(confirming the shared build cache is fresh)".into(),
11926 code: None,
11927 output_tail: e.to_string(),
11928 duration_ms: started.elapsed().as_millis() as u64,
11929 resource_blocked: true,
11930 }],
11931 false,
11932 ));
11933 }
11934 }
11935 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11936 if !timed_out_pids.is_empty() {
11937 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11938 }
11939 drop(_coordination);
11942 if let Some(guard) = guard {
11943 guard.release();
11944 }
11945 Ok((outcomes, retried))
11946}
11947
11948async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11960 wait_for_pids_with(
11961 pids,
11962 crate::proc::pid_alive,
11963 LEASE_RELEASE_POLL,
11964 LEASE_RELEASE_MAX_WAIT,
11965 )
11966 .await;
11967}
11968
11969async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11975 pids: &[u32],
11976 alive: F,
11977 poll: Duration,
11978 max_wait: Duration,
11979) {
11980 let deadline = Instant::now() + max_wait;
11981 loop {
11982 if pids.iter().all(|&pid| !alive(pid)) {
11983 return;
11984 }
11985 if Instant::now() >= deadline {
11986 return;
11987 }
11988 tokio::time::sleep(poll).await;
11989 }
11990}
11991
11992fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11999 outcomes.iter().any(|o| o.resource_blocked)
12000}
12001
12002pub(crate) enum GateFix {
12004 Retry,
12006 Stop,
12009 Defer,
12012}
12013
12014pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12022 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12023 red.peek().is_some()
12024 && red.all(|o| {
12025 !o.resource_blocked
12026 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12027 && !o.output_tail.trim().is_empty()
12028 })
12029}
12030
12031fn e2e_outcome_label(o: &CommandOutcome) -> String {
12035 if o.ok() {
12036 return "pass".to_owned();
12037 }
12038 let reason = if o.build_failed() {
12039 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12040 } else {
12041 format!("FAIL ({:?})", o.code)
12042 };
12043 format!(
12044 "{reason}\n{}",
12045 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12046 )
12047}
12048
12049async fn run_e2e_with_retry(
12057 state: &mut RunState,
12058 shell: &[String],
12059 commands: &[String],
12060 worktree: &Path,
12061 timeout: Duration,
12062 context: &str,
12063) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12064 let (mut e2e, mut timed_out_pids) = run_commands(
12065 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12066 )
12067 .await;
12068 for o in &e2e {
12069 state.event(
12070 "verify",
12071 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12072 );
12073 }
12074 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12078 if verify_retried {
12079 state.event(
12080 "verify",
12081 format!(
12082 "{context}: verify could not build/link, not a test result — retrying once \
12083 before concluding"
12084 ),
12085 );
12086 let retried = run_commands(
12087 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12088 )
12089 .await;
12090 e2e = retried.0;
12091 timed_out_pids.extend(retried.1);
12094 for o in &e2e {
12095 state.event(
12096 "verify",
12097 format!(
12098 "{context}: retry `{}` -> {}",
12099 o.command,
12100 e2e_outcome_label(o)
12101 ),
12102 );
12103 }
12104 }
12105 (e2e, verify_retried, timed_out_pids)
12106}
12107
12108#[allow(clippy::too_many_arguments)]
12124async fn run_commands(
12125 state: &mut RunState,
12126 node: &str,
12127 task: &str,
12128 attempt: usize,
12129 shell: &[String],
12130 commands: &[String],
12131 cwd: &Path,
12132 timeout: Duration,
12133) -> (Vec<CommandOutcome>, Vec<u32>) {
12134 if commands.is_empty() {
12135 return (Vec::new(), Vec::new());
12140 }
12141 let mut out = Vec::new();
12142 let mut timed_out_pids = Vec::new();
12143 let total = commands.len();
12144 for (idx, command) in commands.iter().enumerate() {
12145 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12146 if let Err(e) = state.save() {
12147 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12148 }
12149 let started = Instant::now();
12150 let mut cmd = tokio::process::Command::new(&shell[0]);
12151 cmd.quiet();
12152 for key in crate::worker::INVOCATION_ENV {
12156 cmd.env_remove(key);
12157 }
12158 cmd.args(&shell[1..])
12159 .arg(command)
12160 .current_dir(cwd)
12161 .stdin(std::process::Stdio::null())
12162 .stdout(std::process::Stdio::piped())
12163 .stderr(std::process::Stdio::piped())
12164 .kill_on_drop(true);
12165 let mut outputs = Vec::new();
12166 let protected = (|| -> Result<()> {
12167 crate::incremental::configure(&mut cmd, cwd)?;
12168 let mut paths = vec![cwd.to_owned()];
12169 if let Some(cache) = state.config.cache_dir() {
12170 paths.push(cache);
12171 }
12172 outputs.extend(crate::retention::OutputLease::admit(
12173 &crate::run::home(),
12174 &paths,
12175 state.config.disk.min_free_bytes,
12176 )?);
12177 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12178 outputs.push(guard);
12179 }
12180 if let Some(cache) = state.config.cache_dir() {
12181 outputs.push(crate::retention::OutputLease::cache(
12182 &crate::run::home(),
12183 &cache,
12184 false,
12185 )?);
12186 }
12187 Ok(())
12188 })();
12189 if let Err(e) = protected {
12190 let detail = format!("execution admission unavailable: {e:#}");
12191 state.resource_block = Some(crate::run::ResourceBlock {
12192 cause: crate::run::ResourceBlockCause::Other,
12193 node: node.to_owned(),
12194 kind: if detail.contains("disk admission") {
12195 "disk admission"
12196 } else {
12197 "output ownership"
12198 }
12199 .to_owned(),
12200 detail: detail.clone(),
12201 at: Timestamp::now(),
12202 });
12203 out.push(CommandOutcome {
12204 command: command.clone(),
12205 code: None,
12206 output_tail: detail,
12207 duration_ms: 0,
12208 resource_blocked: true,
12209 });
12210 break;
12211 }
12212 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12213 let (code, body) = match spawned {
12214 Ok(mut running) => {
12215 let pid = running.child().id();
12216 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12217 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12218 let mut paths = vec![cwd.to_owned()];
12219 if let Some(cache) = state.config.cache_dir() {
12220 paths.push(cache);
12221 }
12222 let mut pressure = None;
12223 let status = tokio::select! {
12224 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12225 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12226 pressure = Some(reason); None
12227 }
12228 };
12229 let (code, failure) = match status {
12230 Some(Ok(Ok(status))) => (
12231 status.code(),
12232 status
12233 .code()
12234 .is_none()
12235 .then(|| format!("command terminated: {status}")),
12236 ),
12237 Some(Ok(Err(error))) => {
12238 (None, Some(format!("failed waiting for command: {error}")))
12239 }
12240 Some(Err(_)) => {
12241 if !running.coordinated()
12245 && let Some(pid) = pid
12246 {
12247 timed_out_pids.push(pid);
12248 }
12249 (
12250 None,
12251 Some(format!("timed out after {}s", timeout.as_secs())),
12252 )
12253 }
12254 None => (None, None),
12255 };
12256 running.finish().await;
12257 if let Some(reason) = pressure {
12258 state.resource_block = Some(crate::run::ResourceBlock {
12259 cause: crate::run::ResourceBlockCause::Other,
12260 node: node.to_owned(),
12261 kind: "disk pressure".to_owned(),
12262 detail: reason.clone(),
12263 at: Timestamp::now(),
12264 });
12265 out.push(CommandOutcome {
12266 command: command.clone(),
12267 code: None,
12268 output_tail: reason,
12269 duration_ms: started.elapsed().as_millis() as u64,
12270 resource_blocked: true,
12271 });
12272 break;
12273 }
12274 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12275 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12276 if let Some(failure) = failure {
12277 body.push('\n');
12278 body.push_str(&failure);
12279 }
12280 (code, body)
12281 }
12282 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12283 };
12284 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12285 if !body.is_empty() {
12286 match crate::run::write_verification_artifact(state, task, &body) {
12290 Ok(name) if summary != body => {
12291 summary = format!("[full output: artifacts/{name}]\n{summary}");
12292 }
12293 Ok(_) => {}
12294 Err(e) => {
12295 let warning = format!("could not save the full {task} output: {e:#}");
12296 tracing::warn!("{warning}");
12297 state.event(node, warning);
12298 if summary != body {
12299 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12300 }
12301 }
12302 }
12303 }
12304 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12305 out.push(CommandOutcome {
12306 command: command.clone(),
12307 code,
12308 output_tail: summary,
12309 duration_ms: started.elapsed().as_millis() as u64,
12310 resource_blocked,
12311 });
12312 if resource_blocked {
12313 state.resource_block = Some(crate::run::ResourceBlock {
12314 cause: crate::run::ResourceBlockCause::Other,
12315 node: node.to_owned(),
12316 kind: "incremental cache admission".into(),
12317 detail: body,
12318 at: Timestamp::now(),
12319 });
12320 break;
12321 }
12322 }
12323 state.task_finished(task);
12324 if let Err(e) = state.save() {
12325 tracing::warn!("could not persist the end of {task}: {e:#}");
12326 }
12327 (out, timed_out_pids)
12328}
12329
12330fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12342 let repo = repo.display();
12343 match style {
12344 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12345 MergeStyle::Squash => {
12346 let subject = message
12349 .lines()
12350 .next()
12351 .unwrap_or(branch)
12352 .replace(['\\', '"', '$', '`'], "");
12353 format!(
12354 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12355 )
12356 }
12357 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12358 }
12359}
12360
12361const PR_TITLE_MAX: usize = 240;
12372
12373fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12384 let line = queue::first_line(text)?;
12385 let chars: Vec<char> = line.chars().collect();
12386 let (kept, mark) = if chars.len() <= max {
12387 (line.to_owned(), "")
12388 } else if let Some(end) = sentence_end(&chars, max) {
12389 (chars[..end].iter().collect::<String>(), "")
12390 } else {
12391 let room = max.saturating_sub(3);
12392 let cut = if chars[room].is_whitespace() {
12395 room
12396 } else {
12397 chars[..room]
12398 .iter()
12399 .rposition(|c| c.is_whitespace())
12400 .unwrap_or(room)
12401 };
12402 let head: String = chars[..cut].iter().collect();
12403 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12404 (head.to_owned(), "...")
12405 };
12406 if kept.is_empty() {
12407 return None;
12408 }
12409 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12410 return None;
12411 }
12412 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12413 return None;
12414 }
12415 Some(format!("{kept}{mark}"))
12416}
12417
12418fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12422 const MIN: usize = 20;
12423 for i in MIN..max.min(chars.len()) {
12424 if !matches!(chars[i], '.' | '!' | '?') {
12425 continue;
12426 }
12427 let Some(&next) = chars.get(i + 1) else {
12428 continue;
12429 };
12430 if !next.is_whitespace() {
12431 continue;
12432 }
12433 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12434 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12435 continue;
12436 }
12437 let word: String = chars[..i]
12438 .iter()
12439 .rev()
12440 .take_while(|c| !c.is_whitespace())
12441 .collect::<Vec<_>>()
12442 .into_iter()
12443 .rev()
12444 .collect();
12445 let word = word.to_ascii_lowercase();
12446 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12447 continue;
12448 }
12449 let end = chars[..i]
12450 .iter()
12451 .rposition(|c| !c.is_whitespace())
12452 .map_or(i, |p| p + 1);
12453 return Some(end);
12454 }
12455 None
12456}
12457
12458struct PrMessage {
12463 title: String,
12464 body: String,
12465}
12466
12467impl PrMessage {
12468 fn commit_message(&self) -> String {
12472 format!("{}\n\n{}", self.title, self.body)
12473 }
12474}
12475
12476fn title_marker(line: &str) -> Option<&str> {
12478 let line = line.trim();
12479 let head = line.get(..6)?;
12480 head.eq_ignore_ascii_case("title:")
12481 .then(|| line[6..].trim())
12482}
12483
12484fn summary_title(summary: &str) -> Option<String> {
12489 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12490 let raw = title_marker(first)?;
12491 if raw.is_empty() {
12492 return None;
12493 }
12494 let title = queue::title_from(raw, PR_TITLE_MAX);
12495 let lower = title.to_ascii_lowercase();
12496 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12497 return None;
12498 }
12499 Some(title)
12500}
12501
12502const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12504
12505pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12507
12508pub(crate) fn is_review_run(state: &RunState) -> bool {
12512 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12513}
12514
12515fn review_title(state: &RunState) -> Option<String> {
12520 english_subject(
12521 state.reviewed_commits.as_ref()?.first()?,
12522 state.config.graph.github_english(),
12523 )
12524}
12525
12526fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12529 let raw = raw.trim();
12530 if raw.is_empty() {
12531 return None;
12532 }
12533 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12534 return None;
12535 }
12536 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12537 return None;
12538 }
12539 let title = queue::title_from(raw, PR_TITLE_MAX);
12540 let lower = title.to_ascii_lowercase();
12541 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12542 return None;
12543 }
12544 Some(title)
12545}
12546
12547#[derive(Debug, Clone, PartialEq, Eq)]
12550struct BranchFacts {
12551 commits: Vec<(String, String)>,
12553 stat: String,
12555}
12556
12557const STAT_FILE_LINES: usize = 25;
12559const COMMITS_MAX_CHARS: usize = 20_000;
12561
12562async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12565 let commits = git::commit_log(repo, base, branch).await.ok()?;
12566 if commits.is_empty() {
12567 return None;
12568 }
12569 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12570 let lines: Vec<&str> = stat.lines().collect();
12571 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12572 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12573 let more = format!(" ... {omitted} more file(s)");
12574 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12575 kept.push(&more);
12576 kept.push(lines[lines.len() - 1]);
12577 kept.join("\n")
12578 } else {
12579 lines.join("\n")
12580 };
12581 Some(BranchFacts { commits, stat })
12582}
12583
12584fn markdown_safe(text: &str) -> String {
12587 text.replace("</details>", "</details>")
12588 .replace("\x60\x60\x60", "~~~")
12589}
12590
12591fn neutral_title(state: &RunState, winner: char) -> String {
12592 format!(
12593 "chore: land candidate {} of run {}",
12594 winner.to_ascii_uppercase(),
12595 state.id
12596 )
12597}
12598
12599pub(crate) async fn repair_landing_gate(
12602 state: &mut RunState,
12603 winner: &Candidate,
12604 outcomes: &[CommandOutcome],
12605) -> Result<GateFix> {
12606 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12607 return Ok(GateFix::Stop);
12608 }
12609 let mut runner = Runner {
12610 roles: state.config.resolve_roles()?,
12611 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12612 state: state.clone(),
12613 pause: Pause::new(),
12614 interrupt: Pause::new(),
12615 };
12616 let result = runner.gate_fix_round(winner, outcomes).await;
12617 *state = runner.state;
12618 result
12619}
12620
12621pub(crate) async fn verify_current_head(
12624 state: &mut RunState,
12625 winner: &Candidate,
12626 wait: &crate::repo_lock::WaitPolicy,
12627) -> Result<bool> {
12628 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12629 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12631 state
12632 .verification_history
12633 .push(crate::run::VerificationEvidence {
12634 superseded_by: Some(head.clone()),
12635 archived_at: Some(Timestamp::now()),
12636 fixes_spent: Some(state.gate_fixes.len()),
12637 gate_head: None,
12638 gate_ran: false,
12639 gate: Vec::new(),
12640 e2e: state.head_e2e.take(),
12641 });
12642 }
12643 if let Some(e) = &state.head_e2e
12644 && e.head == head
12645 {
12646 if e.passed {
12647 return Ok(true);
12648 }
12649 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12650 state.status = RunStatus::Blocked;
12651 state.save()?;
12652 return Ok(false);
12653 }
12654 } else if state.reviews.last().is_some_and(|r| {
12655 r.verified_head.as_deref() == Some(head.as_str())
12656 && (r.e2e_status() == E2eStatus::Passed
12657 || (r.e2e_status() == E2eStatus::NotConfigured
12658 && state.config.verify.e2e.is_empty()))
12659 }) || state
12660 .gate_repair_e2e
12661 .as_ref()
12662 .is_some_and(|e| e.head == head && e.passed)
12663 {
12664 return Ok(true);
12665 }
12666 if let Some(e) = &state.gate_repair_e2e
12667 && e.head == head
12668 && !e.passed
12669 && !e.outcomes.is_empty()
12670 && !verify_inconclusive(&e.outcomes)
12671 {
12672 state.status = RunStatus::Blocked;
12673 state.save()?;
12674 return Ok(false);
12675 }
12676 state.head_e2e = Some(crate::run::GateRepairE2e {
12677 head: head.clone(),
12678 passed: false,
12679 outcomes: Vec::new(),
12680 });
12681 state.save()?;
12682 let commands = state.config.verify.e2e.clone();
12683 let shell = state.config.shell();
12684 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12685 let cache_dir = state.config.cache_dir();
12686 let outcomes = if commands.is_empty() {
12687 Vec::new()
12688 } else {
12689 let (outcomes, _) = with_cache_lease(
12690 state,
12691 cache_dir.as_deref(),
12692 "e2e",
12693 "e2e",
12694 &winner.worktree,
12695 &head,
12696 timeout,
12697 "current head e2e",
12698 wait,
12699 |state, budget| async move {
12700 run_e2e_with_retry(
12701 state,
12702 &shell,
12703 &commands,
12704 &winner.worktree,
12705 budget,
12706 "current head e2e",
12707 )
12708 .await
12709 },
12710 )
12711 .await?;
12712 outcomes
12713 };
12714 let unresolved = verify_inconclusive(&outcomes);
12715 let passed = !unresolved
12716 && outcomes.iter().all(CommandOutcome::ok)
12717 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12718 if !outcomes.is_empty() && !unresolved {
12719 state.record_base_progress(&head);
12720 }
12721 state.head_e2e = Some(crate::run::GateRepairE2e {
12722 head: head.clone(),
12723 passed,
12724 outcomes,
12725 });
12726 if !passed && !unresolved {
12727 state.status = RunStatus::Blocked;
12728 state.event(
12729 "e2e",
12730 format!("verify.e2e failed on head {head}; nothing published"),
12731 );
12732 }
12733 state.save()?;
12734 Ok(passed)
12735}
12736
12737pub(crate) async fn check_gate_repair_e2e(
12740 state: &mut RunState,
12741 winner: &Candidate,
12742 wait: &crate::repo_lock::WaitPolicy,
12743) -> Result<GateFix> {
12744 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12745 return Ok(GateFix::Retry);
12746 }
12747 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12748 if state
12749 .gate_repair_e2e
12750 .as_ref()
12751 .is_some_and(|r| r.head == head && r.passed)
12752 {
12753 return Ok(GateFix::Retry);
12754 }
12755 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12756 head: head.clone(),
12757 passed: false,
12758 outcomes: Vec::new(),
12759 });
12760 state.save()?;
12761 let commands = state.config.verify.e2e.clone();
12762 let outcomes = if commands.is_empty() {
12763 Vec::new()
12764 } else {
12765 let shell = state.config.shell();
12766 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12767 let cache_dir = state.config.cache_dir();
12768 let (outcomes, _) = with_cache_lease(
12769 state,
12770 cache_dir.as_deref(),
12771 "e2e",
12772 "e2e",
12773 &winner.worktree,
12774 &head,
12775 timeout,
12776 "gate repair e2e",
12777 wait,
12778 |state, budget| async move {
12779 run_e2e_with_retry(
12780 state,
12781 &shell,
12782 &commands,
12783 &winner.worktree,
12784 budget,
12785 "gate repair e2e",
12786 )
12787 .await
12788 },
12789 )
12790 .await?;
12791 outcomes
12792 };
12793 let deferred = verify_inconclusive(&outcomes);
12794 let passed = outcomes.iter().all(CommandOutcome::ok)
12795 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12796 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12797 head,
12798 passed,
12799 outcomes,
12800 });
12801 state.save()?;
12802 if deferred {
12803 Ok(GateFix::Defer)
12804 } else if !passed {
12805 state.event(
12806 "gate",
12807 "verify.e2e failed after gate repair; nothing published",
12808 );
12809 Ok(GateFix::Stop)
12810 } else {
12811 Ok(GateFix::Retry)
12812 }
12813}
12814
12815pub(crate) async fn run_landing_gate(
12817 state: &mut RunState,
12818 winner: &Candidate,
12819 wait: &crate::repo_lock::WaitPolicy,
12820) -> Result<Vec<CommandOutcome>> {
12821 let shell = state.config.shell();
12822 let gate_commands = state.config.verify.gate.clone();
12823 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12824 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12825 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12826 Vec::new()
12827 } else {
12828 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12829 let cache_dir = state.config.cache_dir();
12830 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12831 let (outcomes, _) = with_cache_lease(
12832 state,
12833 cache_dir.as_deref(),
12834 "gate",
12835 "gate",
12836 &winner.worktree,
12837 &head,
12838 timeout,
12839 "final gate",
12840 wait,
12841 |state, budget| {
12842 let shell = shell.clone();
12843 let gate_commands = gate_commands.clone();
12844 let winner = winner.clone();
12845 async move {
12846 Runner::run_pre_gate(state, &winner).await;
12847 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12849 let (outcomes, timed_out_pids) = run_commands(
12850 state,
12851 "gate",
12852 "gate",
12853 0,
12854 &shell,
12855 &gate_commands,
12856 &winner.worktree,
12857 budget,
12858 )
12859 .await;
12860 (outcomes, false, timed_out_pids)
12861 }
12862 },
12863 )
12864 .await?;
12865 outcomes
12866 };
12867 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12868 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12869 let mut outcomes = outcomes;
12870 outcomes.push(CommandOutcome {
12871 command: "gate head stability".into(), code: None,
12872 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12873 duration_ms: 0, resource_blocked: false,
12874 });
12875 return Ok(outcomes);
12876 }
12877 if outcomes.is_empty() {
12878 state.event("gate", "no gate commands configured; not tested");
12883 }
12884 for o in &outcomes {
12885 state.event(
12886 "gate",
12887 format!(
12888 "`{}` -> {}",
12889 o.command,
12890 if o.ok() {
12891 "pass".to_owned()
12892 } else {
12893 format!(
12894 "FAIL ({:?})\n{}",
12895 o.code,
12896 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12897 )
12898 }
12899 ),
12900 );
12901 }
12902 Ok(outcomes)
12903}
12904
12905pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12911 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12912 ""
12913 } else {
12914 pr_title
12915 }
12916}
12917
12918pub fn landing_subject_source(state: &RunState) -> String {
12922 if is_review_run(state) {
12923 let winner = state.candidates.first().map_or('A', |c| c.label);
12924 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12925 }
12926 state.instruction.clone()
12927}
12928
12929fn summary_without_title(summary: &str) -> String {
12932 let mut lines = summary.trim().lines().peekable();
12933 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12934 lines.next();
12935 }
12936 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12937}
12938
12939#[cfg(test)]
12951fn pr_message(state: &RunState, winner: char) -> PrMessage {
12952 pr_message_with(state, winner, None)
12953}
12954
12955#[cfg(test)]
12958fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12959 let raw = pr_message_raw(
12960 state,
12961 winner,
12962 facts,
12963 crate::pr_outcome::FinalHead::Unreported,
12964 ChangeNote::NotAsked,
12965 );
12966 let id = crate::scrub::Identity::current();
12967 PrMessage {
12968 title: crate::scrub::scrub(&raw.title, &id),
12969 body: crate::scrub::scrub(&raw.body, &id),
12970 }
12971}
12972
12973#[derive(Debug, Clone, Copy)]
12975enum ChangeNote<'a> {
12976 NotAsked,
12978 Summary(&'a crate::run::ChangeSummary),
12980 Unavailable(&'a str),
12982}
12983
12984fn pr_message_raw(
12985 state: &RunState,
12986 winner: char,
12987 facts: Option<&BranchFacts>,
12988 head: crate::pr_outcome::FinalHead<'_>,
12989 change: ChangeNote<'_>,
12990) -> PrMessage {
12991 if !matches!(change, ChangeNote::NotAsked) {
12992 let (title, mut body) = match change {
12993 ChangeNote::Summary(c) => (
12994 c.title.clone(),
12995 format!("## Summary\n\n{}\n\n", c.body.trim()),
12996 ),
12997 ChangeNote::Unavailable(why) => (
12998 neutral_title(state, winner),
12999 format!(
13000 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13001 ),
13002 ),
13003 ChangeNote::NotAsked => unreachable!(),
13004 };
13005 body.push_str(&public_pr_evidence(state, head));
13006 body.push_str(&format!(
13007 "\n---\nmagi:run/{} magi:candidate-{}\n",
13008 state.id,
13009 winner.to_ascii_lowercase()
13010 ));
13011 return PrMessage { title, body };
13012 }
13013 pr_preview(state, winner, facts, head)
13014}
13015
13016fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13019 let mut body = crate::pr_outcome::review_rounds(state);
13020 body.push_str(&crate::pr_outcome::final_verification(state, head));
13021 let open = state.last_round_findings();
13022 if !open.is_empty() {
13023 body.push_str("## Open review findings\n\n");
13024 for f in open {
13025 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13026 }
13027 body.push('\n');
13028 }
13029 if let Some(c) = state.change_summary.as_ref()
13030 && head == crate::pr_outcome::FinalHead::At(&c.head)
13031 {
13032 body.push_str(&format!(
13033 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13034 c.base, c.head
13035 ));
13036 }
13037 body
13038}
13039
13040fn pr_preview(
13042 state: &RunState,
13043 winner: char,
13044 facts: Option<&BranchFacts>,
13045 head: crate::pr_outcome::FinalHead<'_>,
13046) -> PrMessage {
13047 let summary = state
13048 .candidates
13049 .iter()
13050 .find(|c| c.label == winner)
13051 .map(|c| c.summary.as_str())
13052 .unwrap_or_default();
13053 let review = is_review_run(state);
13056 let title = if review {
13057 facts
13058 .and_then(|f| {
13059 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13060 })
13061 .or_else(|| review_title(state))
13062 .or_else(|| {
13063 state
13064 .candidates
13065 .iter()
13066 .find(|c| c.label == winner)
13067 .filter(|c| !c.branch.starts_with("magi/"))
13068 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13069 })
13070 .unwrap_or_else(|| neutral_title(state, winner))
13071 } else {
13072 summary_title(summary).unwrap_or_else(|| {
13073 english_title(
13074 &state.instruction,
13075 PR_TITLE_MAX,
13076 state.config.graph.github_english(),
13077 )
13078 .unwrap_or_else(|| neutral_title(state, winner))
13079 })
13080 };
13081
13082 let mut body = String::new();
13083 let what = summary_without_title(summary);
13084 if !what.is_empty() {
13085 body.push_str("## Summary\n\n");
13086 body.push_str(&what);
13087 body.push_str("\n\n");
13088 }
13089
13090 body.push_str(&crate::pr_outcome::review_rounds(state));
13092 body.push_str(&crate::pr_outcome::final_verification(state, head));
13093 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13094
13095 let open = state.open_findings();
13096 if !open.is_empty() {
13097 body.push_str("## Open review findings\n\n");
13098 for f in &open {
13099 body.push_str(&format!(
13100 "- `{}` [{:?}] {}\n",
13101 f.id, f.severity, "Context retained in local review records."
13102 ));
13103 }
13104 body.push('\n');
13105 }
13106
13107 if let Some(fix) = fix
13108 && !fix.rejected.is_empty()
13109 {
13110 body.push_str("## Declined by the fixer\n\n");
13111 for r in &fix.rejected {
13112 body.push_str(&format!(
13113 "- `{}`: {}\n",
13114 r.id, "Reason retained in local review records."
13115 ));
13116 }
13117 body.push('\n');
13118 }
13119
13120 if review {
13121 body.push_str("## Commits under review\n\n");
13123 if let Some(facts) = facts {
13124 let mut left = COMMITS_MAX_CHARS;
13125 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13126 let mut entry = format!("- {}\n", markdown_safe(subject));
13127 for l in markdown_safe(text).lines() {
13128 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13129 }
13130 if left == 0 {
13131 body.push_str(&format!(
13132 "- ... {} more commit(s)\n",
13133 facts.commits.len() - i
13134 ));
13135 break;
13136 }
13137 if entry.len() > left {
13138 let mut end = left;
13141 while !entry.is_char_boundary(end) {
13142 end -= 1;
13143 }
13144 entry.truncate(end);
13145 entry.push_str("\n ... (truncated)\n");
13146 left = 0;
13147 } else {
13148 left -= entry.len();
13149 }
13150 body.push_str(&entry);
13151 }
13152 if !facts.stat.trim().is_empty() {
13153 body.push_str(&format!(
13154 "\n## Diff stat\n\n```\n{}\n```\n",
13155 markdown_safe(facts.stat.trim())
13156 ));
13157 }
13158 } else {
13159 match &state.reviewed_commits {
13160 Some(subjects) => {
13161 for s in subjects {
13162 body.push_str(&format!("- {}\n", s.trim()));
13163 }
13164 }
13165 None => {
13166 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13169 body.push_str(rest.trim());
13170 body.push('\n');
13171 }
13172 }
13173 }
13174 } else {
13175 let task = state.instruction.trim();
13176 let task = if task.is_empty() {
13177 "(empty task)"
13178 } else {
13179 task
13180 };
13181 body.push_str(&format!(
13182 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13183 task.replace("</details>", "</details>")
13184 ));
13185 }
13186
13187 body.push_str(&format!(
13188 "\n---\nmagi:run/{} magi:candidate-{}\n",
13189 state.id,
13190 winner.to_ascii_lowercase()
13191 ));
13192
13193 PrMessage { title, body }
13194}
13195
13196fn seeded_instruction(state: &RunState) -> String {
13199 let mut inputs = Vec::new();
13200 if let Some(start) = &state.sources.start_from {
13201 inputs.push(format!(
13202 "Start from `{}` at `{}` (including ancestry).",
13203 start.token, start.sha
13204 ));
13205 }
13206 for pick in &state.sources.pick_commits {
13207 inputs.push(format!(
13208 "Pick `{}` at `{}` (single commit).",
13209 pick.token, pick.sha
13210 ));
13211 }
13212 if inputs.is_empty() {
13213 state.instruction.clone()
13214 } else {
13215 format!(
13216 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13217 state.instruction,
13218 inputs.join("\n")
13219 )
13220 }
13221}
13222
13223async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13227 let base = &state.base_branch;
13228 let mut against = base.clone();
13229 if mode == MergeMode::Pr {
13230 let remote = &state.config.merge.remote;
13234 let tracking = format!("{remote}/{base}");
13235 let fetched = git::fetch(repo, remote, base).await;
13236 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13237 against = tracking;
13238 } else {
13239 return false;
13240 }
13241 }
13242 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13243}
13244
13245fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13248 let mut detail = format!(
13249 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13250 nothing to open a pull request for"
13251 );
13252 detail.push_str(
13253 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13254 );
13255 detail
13256}
13257
13258#[derive(Debug, PartialEq, Eq)]
13261enum PrPlan {
13262 Create,
13263 Adopt { url: String, title: String },
13264 Stop(String),
13265}
13266
13267fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13270 match found {
13271 Ok(land::OpenPr::None) => PrPlan::Create,
13272 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13273 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13274 "several open pull requests exist for this branch, not picking one: {}",
13275 urls.join(" ")
13276 )),
13277 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13278 }
13279}
13280
13281pub(crate) async fn gh_pr_create(
13283 cwd: &Path,
13284 base: &str,
13285 head: &str,
13286 title: &str,
13287 body: &str,
13288) -> Result<String> {
13289 let out = tokio::process::Command::new("gh")
13290 .args([
13291 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13292 ])
13293 .current_dir(cwd)
13294 .quiet()
13295 .stdin(std::process::Stdio::null())
13296 .output()
13297 .await
13298 .context("spawn gh")?;
13299 if out.status.success() {
13300 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13301 } else {
13302 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13303 }
13304}
13305
13306async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13308 match std::fs::symlink_metadata(path) {
13309 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13310 git::worktree_remove_clean(repo, path).await?;
13313 return Ok(());
13314 }
13315 Err(e) => return Err(e.into()),
13316 Ok(_) => {}
13317 }
13318 if !git::is_clean(path).await? {
13319 bail!(
13320 "worktree has uncommitted work; refusing removal: {}",
13321 path.display()
13322 );
13323 }
13324 if !git::worktree_remove_clean(repo, path).await? {
13325 bail!("worktree removal refused: {}", path.display());
13326 }
13327 Ok(())
13328}
13329
13330pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13339 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13340 bail!("refusing fold: {reason}");
13341 }
13342 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13343 bail!("candidate preservation incomplete; refusing cleanup");
13344 }
13345 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13349 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13350 let _driver = crate::driver::maintenance(home, &state.id)?;
13351 let mut output_guards = Vec::new();
13352 for entry in std::fs::read_dir(state.worktree_root())
13353 .into_iter()
13354 .flatten()
13355 .flatten()
13356 {
13357 if entry.path().join(".git").is_file() {
13358 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13359 }
13360 }
13361 let repo = state.repo.clone();
13362 let wait = repository_wait(state, None);
13365 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13366 Ok(lease) => lease,
13367 Err(e) => {
13368 state.save_under(home)?;
13369 return Err(e);
13370 }
13371 };
13372 let _source_outputs = if state.status == RunStatus::Merged
13375 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13376 {
13377 verify_merged_fold_sources(state, drop_winner, home).await?
13378 } else {
13379 Vec::new()
13380 };
13381 let root = state.worktree_root();
13382 let winner = state.tally.as_ref().map(|t| t.winner);
13383 let mut removed = Vec::new();
13384
13385 for i in 0..state.candidates.len() {
13386 let c = state.candidates[i].clone();
13387 let is_winner = Some(c.label) == winner;
13388 if is_winner && !drop_winner {
13389 continue;
13390 }
13391 let existed = c.worktree.exists();
13392 remove_preserved_worktree(&repo, &c.worktree).await?;
13395 if existed {
13396 removed.push(c.worktree.to_string_lossy().into_owned());
13397 }
13398 let handed_over = state.released_branches.contains(&c.branch);
13401 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13402 if !git::branch_delete(&repo, &c.branch).await? {
13403 bail!("candidate branch removal refused");
13404 }
13405 removed.push(c.branch.clone());
13406 }
13407 state.candidates[i].folded = true;
13408 }
13409
13410 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13411 let path = name.path();
13412 let keep = !drop_winner
13413 && winner.is_some_and(|w| {
13414 path.file_name()
13415 .is_some_and(|n| n == format!("cand-{w}").as_str())
13416 });
13417 if keep {
13418 continue;
13419 }
13420 remove_preserved_worktree(&repo, &path).await?;
13421 removed.push(path.to_string_lossy().into_owned());
13422 }
13423
13424 remove_if_empty(&root);
13433
13434 if state.enabled_worktree_config && drop_winner {
13435 git::release_worktree_config(&repo).await.ok();
13439 state.enabled_worktree_config = false;
13440 }
13441 state.save_under(home)?;
13442 Ok(removed)
13443}
13444
13445async fn verify_merged_fold_sources(
13446 state: &RunState,
13447 drop_winner: bool,
13448 home: &Path,
13449) -> Result<Vec<std::fs::File>> {
13450 let winner = state.tally.as_ref().map(|t| t.winner);
13451 let mut heads = BTreeSet::new();
13452 for candidate in &state.candidates {
13453 if Some(candidate.label) == winner && !drop_winner {
13454 continue;
13455 }
13456 anyhow::ensure!(
13457 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13458 "candidate recovery source is outside the inventoried run root: {}",
13459 candidate.worktree.display()
13460 );
13461 match std::fs::symlink_metadata(&candidate.worktree) {
13462 Ok(_) => {}
13463 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13464 Err(error) => return Err(error.into()),
13465 }
13466 if !state.released_branches.contains(&candidate.branch)
13467 && git::branch_exists(&state.repo, &candidate.branch).await?
13468 {
13469 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13470 }
13471 }
13472 let mut cargo_guards = Vec::new();
13473 let entries = match std::fs::read_dir(state.worktree_root()) {
13474 Ok(entries) => Some(entries),
13475 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13476 Err(error) => return Err(error.into()),
13477 };
13478 for entry in entries.into_iter().flatten() {
13479 let path = entry?.path();
13480 let metadata = std::fs::symlink_metadata(&path)?;
13481 anyhow::ensure!(
13482 metadata.is_dir() && !metadata.file_type().is_symlink(),
13483 "non-directory or symlink recovery source retained: {}",
13484 path.display()
13485 );
13486 if !drop_winner
13487 && winner.is_some_and(|w| {
13488 path.file_name()
13489 .is_some_and(|name| name == format!("cand-{w}").as_str())
13490 })
13491 {
13492 continue;
13493 }
13494 anyhow::ensure!(
13495 path.join(".git").is_file(),
13496 "unregistered recovery source retained: {}",
13497 path.display()
13498 );
13499 anyhow::ensure!(
13500 git::git(
13501 &path,
13502 &[
13503 "status",
13504 "--porcelain",
13505 "--untracked-files=all",
13506 "--ignored",
13507 "--",
13508 ".",
13509 ":(exclude)target"
13510 ]
13511 )
13512 .await?
13513 .is_empty(),
13514 "source changes or ignored recovery files retained: {}",
13515 path.display()
13516 );
13517 match std::fs::symlink_metadata(path.join("target")) {
13518 Ok(_) => {
13519 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13520 }
13521 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13522 Err(error) => return Err(error.into()),
13523 }
13524 for checkpoint in [
13525 "rebase-merge",
13526 "rebase-apply",
13527 "sequencer",
13528 "CHERRY_PICK_HEAD",
13529 "MERGE_HEAD",
13530 "REBASE_HEAD",
13531 "REVERT_HEAD",
13532 "BISECT_LOG",
13533 "AUTO_MERGE",
13534 "MERGE_MSG",
13535 ] {
13536 let location = git::git(
13537 &path,
13538 &[
13539 "rev-parse",
13540 "--path-format=absolute",
13541 "--git-path",
13542 checkpoint,
13543 ],
13544 )
13545 .await?;
13546 anyhow::ensure!(
13547 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13548 if e.kind() == std::io::ErrorKind::NotFound),
13549 "Git recovery checkpoint retained: {checkpoint}"
13550 );
13551 }
13552 heads.insert(git::rev_parse(&path, "HEAD").await?);
13553 }
13554 crate::fold_evidence::preserve(state, home, &heads).await?;
13555
13556 Ok(cargo_guards)
13557}
13558
13559fn remove_if_empty(dir: &Path) {
13570 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13571 std::fs::remove_dir(dir).ok();
13572 }
13573}
13574
13575pub fn worst_open(state: &RunState) -> Option<Severity> {
13577 state
13578 .reviews
13579 .last()?
13580 .reviews
13581 .iter()
13582 .flat_map(|r| r.findings.iter())
13583 .map(|f| f.severity)
13584 .max()
13585}
13586
13587async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13591 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13592 git::acquire_worktree_config(repo).await?;
13593 state.enabled_worktree_config = true;
13594 }
13595 Ok(())
13596}
13597
13598#[cfg(test)]
13599mod tests {
13600 #[tokio::test]
13601 async fn launcher_base_fetch_requires_remote_success() {
13602 let tmp = tempfile::tempdir().unwrap();
13603 let repo = tmp.path().join("repo");
13604 let remote = tmp.path().join("remote");
13605 std::fs::create_dir(&repo).unwrap();
13606 std::fs::create_dir(&remote).unwrap();
13607 git::git(&remote, &["init", "--bare", "-b", "main"])
13608 .await
13609 .unwrap();
13610 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13611 git::git(
13612 &repo,
13613 &[
13614 "-c",
13615 "user.name=Fixture",
13616 "-c",
13617 "user.email=fixture@example.invalid",
13618 "commit",
13619 "--allow-empty",
13620 "-m",
13621 "fixture",
13622 ],
13623 )
13624 .await
13625 .unwrap();
13626 git::git(
13627 &repo,
13628 &["remote", "add", "origin", remote.to_str().unwrap()],
13629 )
13630 .await
13631 .unwrap();
13632 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13633 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13634 assert_eq!(
13635 resolve_base(&repo, "main", "origin", None)
13636 .await
13637 .unwrap()
13638 .commit,
13639 head
13640 );
13641 #[cfg(unix)]
13642 {
13643 let policy: crate::config::Coordination = toml::from_str(
13644 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13645 ).unwrap();
13646 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13647 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13648 let held = crate::repo_lock::SeatLease::operation_with(
13649 &repo,
13650 Some(&policy),
13651 &crate::repo_lock::WaitPolicy::default(),
13652 )
13653 .await
13654 .unwrap();
13655 assert!(held.is_some());
13656 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13657 tokio::pin!(fetch);
13658 assert!(
13659 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13660 .await
13661 .is_err()
13662 );
13663 assert!(
13664 !repo.join(".git/FETCH_HEAD").exists(),
13665 "no fetch before admission"
13666 );
13667 drop(held);
13668 let fetched = fetch.await.unwrap();
13669 assert_eq!(fetched.commit, head);
13670 assert_eq!(fetched.remote, "origin");
13671 assert_eq!(fetched.branch, "main");
13672 }
13673 git::git(
13675 &repo,
13676 &[
13677 "remote",
13678 "set-url",
13679 "origin",
13680 tmp.path().join("missing").to_str().unwrap(),
13681 ],
13682 )
13683 .await
13684 .unwrap();
13685 let error = resolve_base(&repo, "main", "origin", None)
13686 .await
13687 .unwrap_err()
13688 .to_string();
13689 assert!(error.contains("cannot read origin/main"));
13690 assert!(error.contains("refusing to branch off"));
13691 }
13692
13693 #[test]
13694 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13695 let leaked = vec!["chore(deps): update a crate".to_owned()];
13696 let own = "fix(daemon): apply a chosen action";
13697 assert!(should_retitle("", own, &leaked));
13698 assert!(should_retitle(
13699 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13700 own,
13701 &leaked
13702 ));
13703 assert!(should_retitle(
13704 "chore: land candidate A of run 1",
13705 own,
13706 &leaked
13707 ));
13708 assert!(should_retitle(
13709 "magi: candidate A (uncommitted work)",
13710 own,
13711 &leaked
13712 ));
13713 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13714 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13715 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13716 }
13717
13718 #[test]
13719 fn pr_merge_plan_creates_adopts_or_stops() {
13720 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13721 assert_eq!(
13722 pr_merge_plan(Ok(land::OpenPr::One {
13723 url: "u".into(),
13724 title: "t".into()
13725 })),
13726 PrPlan::Adopt {
13727 url: "u".into(),
13728 title: "t".into()
13729 }
13730 );
13731 let PrPlan::Stop(many) =
13732 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13733 else {
13734 panic!("many must stop");
13735 };
13736 assert!(many.contains('a') && many.contains('b'));
13737 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13738 panic!("a failed lookup must stop");
13739 };
13740 assert!(err.contains("bad token"));
13741 }
13742
13743 use super::*;
13744 use crate::run::GateStatus;
13745 use std::collections::BTreeMap;
13746 use std::time::Duration;
13747
13748 fn conductor() -> AgentSpec {
13749 AgentSpec {
13750 id: "conductor".to_owned(),
13751 kind: crate::config::AgentKind::Command,
13752 model: None,
13753 command: vec!["true".to_owned()],
13754 extra_args: Vec::new(),
13755 env: BTreeMap::new(),
13756 prompt_delivery: None,
13757 max_turn_secs: None,
13758 }
13759 }
13760
13761 fn spec(id: &str) -> AgentSpec {
13762 AgentSpec {
13763 id: id.to_owned(),
13764 kind: crate::config::AgentKind::Command,
13765 model: None,
13766 command: vec!["true".to_owned()],
13767 extra_args: Vec::new(),
13768 env: BTreeMap::new(),
13769 prompt_delivery: None,
13770 max_turn_secs: None,
13771 }
13772 }
13773
13774 fn ids(xs: &[&str]) -> BTreeSet<String> {
13775 xs.iter().map(|s| (*s).to_owned()).collect()
13776 }
13777
13778 #[test]
13779 fn pick_successor_skips_an_agent_another_seat_holds() {
13780 let roster = [spec("a"), spec("b"), spec("c")];
13782 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13783 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13784 }
13785
13786 #[test]
13787 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13788 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13790 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13791 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13792 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13793 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13794 }
13795
13796 #[test]
13797 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13798 let roster = [spec("a"), spec("b")];
13799 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13800 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13801 let carried = ids(&["b"]);
13802 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13803 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13804 }
13805
13806 #[test]
13807 fn pick_successor_returns_none_without_any_untried_successor() {
13808 let roster = [spec("a"), spec("b")];
13809 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13810 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13811 let carried = ids(&["a"]);
13812 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13813 }
13814
13815 #[test]
13816 fn pick_successor_rescue_avoids_another_seats_occupant() {
13817 let roster = [spec("a"), spec("b"), spec("c")];
13819 let carried = ids(&["b", "c"]);
13820 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13821 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13822 }
13823
13824 #[test]
13825 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13826 let roster = [spec("a"), spec("b"), spec("c")];
13827 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13828 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13829 }
13830
13831 #[test]
13832 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13833 let roster = [spec("a"), spec("b"), spec("c")];
13834 let failed = ids(&["a", "b", "c"]);
13835 let mut tried = ids(&["b"]);
13837 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13838 assert_eq!(first.id, "a");
13839 tried.insert(first.id.clone());
13840 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13841 assert_eq!(second.id, "c");
13842 tried.insert(second.id.clone());
13843 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13844 }
13845
13846 #[test]
13847 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13848 let roster = [spec("a"), spec("b")];
13849 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13850 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13851 }
13852
13853 #[test]
13854 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13855 let roster = [spec("a"), spec("b"), spec("c")];
13856 let h = SeatHistory {
13857 failed: ids(&["a"]),
13858 last_ok: Some("b".to_owned()),
13859 last_fail: None,
13860 };
13861 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13862 let gone = SeatHistory {
13864 last_ok: Some("zzz".to_owned()),
13865 ..h.clone()
13866 };
13867 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13868 let failed = SeatHistory {
13869 failed: ids(&["a", "b"]),
13870 last_ok: Some("b".to_owned()),
13871 last_fail: None,
13872 };
13873 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13874 }
13875
13876 #[test]
13877 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13878 let first = SeatState::new("review-1", "alpha", 7);
13879 let next = handover_seat("review-1", "gamma", 7);
13880 assert_ne!(first.claude_session, next.claude_session);
13881 }
13882
13883 #[test]
13884 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13885 let mut state = state_with_summary("x", "y");
13886 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13887 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13888 assert_ne!(a.claude_session, b.claude_session);
13889 }
13890
13891 #[test]
13892 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13893 let roster = [spec("a"), spec("b")];
13894 let h = SeatHistory {
13895 failed: ids(&["a", "b"]),
13896 ..SeatHistory::default()
13897 };
13898 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13899 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13900 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13901 }
13902
13903 #[test]
13910 fn failure_signature_ignores_numbers_and_paths() {
13911 assert_eq!(
13912 failure_signature("exited with Some(2) and no usable output"),
13913 failure_signature("exited with Some(137) and no usable output")
13914 );
13915 assert_eq!(
13916 failure_signature("process exited with no usable output (exit code 2)"),
13917 failure_signature("process exited with no usable output (exit code 137)")
13918 );
13919 for (old, new) in [
13920 (
13921 "exited with Some(2) and no usable output",
13922 "process exited with no usable output (exit code 2)",
13923 ),
13924 (
13925 "exited with None and no usable output",
13926 "process exited with no usable output (exit code unavailable)",
13927 ),
13928 ] {
13929 let previous = FailClass::Other(failure_signature(old));
13930 let current = FailClass::Other(failure_signature(new));
13931 assert_eq!(previous, current, "resumption retains the failure class");
13932 assert!(!should_hand_over(Some(&previous), ¤t));
13933 }
13934 assert_eq!(
13935 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13936 failure_signature("cannot open /var/x.txt: denied")
13937 );
13938 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13939 }
13940
13941 #[test]
13942 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13943 let other = FailClass::Other("x".into());
13944 assert!(should_hand_over(None, &FailClass::Quota));
13945 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13946 assert!(should_hand_over(
13947 Some(&FailClass::Timeout),
13948 &FailClass::Timeout
13949 ));
13950 assert!(should_hand_over(None, &other));
13951 assert!(!should_hand_over(Some(&other), &other));
13952 assert!(should_hand_over(
13953 Some(&other),
13954 &FailClass::Other("y".into())
13955 ));
13956 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13958 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13959 }
13960
13961 #[test]
13962 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13963 let a = SeatState::new("judge-1", "alpha", 7);
13964 let b = handover_seat("judge-1", "beta", 7);
13965 assert_ne!(a.claude_session, b.claude_session);
13966 assert_eq!(b.turns, 0);
13967 }
13968
13969 #[test]
13970 fn a_timeout_is_classified_apart_from_other_failures() {
13971 assert_eq!(
13972 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13973 Some(FailClass::Timeout)
13974 );
13975 assert!(matches!(
13976 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13977 Some(FailClass::Other(_))
13978 ));
13979 }
13980
13981 #[test]
13982 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13983 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13984 let tried = BTreeSet::from(["beta".to_owned()]);
13985 let next = next_untried_in_roster(&roster, 1, &tried);
13988 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13989 }
13990
13991 #[test]
13992 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13993 let roster = vec![spec("alpha"), spec("beta")];
13994 let tried = BTreeSet::from(["beta".to_owned()]);
13995 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13999 }
14000
14001 #[test]
14002 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14003 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14004 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14005 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14009 }
14010
14011 #[test]
14012 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14013 let roster = vec![spec("a"), spec("a"), spec("b")];
14014 let tried = BTreeSet::from(["a".to_owned()]);
14015 let next = next_untried_in_roster(&roster, 0, &tried);
14016 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14017 }
14018
14019 #[test]
14020 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14021 let roster = vec![spec("a"), spec("b")];
14022 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14023 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14024 }
14025
14026 #[test]
14027 fn remove_if_empty_only_ever_takes_a_bare_directory() {
14028 let dir = tempfile::tempdir().unwrap();
14029 let bay = dir.path().join("ffff");
14030
14031 remove_if_empty(&bay);
14033 assert!(!bay.exists());
14034
14035 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14038 remove_if_empty(&bay);
14039 assert!(bay.exists(), "non-empty directory must survive");
14040
14041 std::fs::remove_dir(bay.join("cand-A")).unwrap();
14043 remove_if_empty(&bay);
14044 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14045 }
14046
14047 #[test]
14056 fn a_full_panel_that_found_nothing_is_clean() {
14057 assert!(round_is_clean(
14058 0,
14059 true,
14060 2,
14061 2,
14062 0,
14063 IncompleteReviewPolicy::Block
14064 ));
14065 }
14066
14067 #[test]
14068 fn a_missing_seat_is_never_clean_under_the_default_policy() {
14069 assert!(!round_is_clean(
14070 0,
14071 true,
14072 1,
14073 2,
14074 0,
14075 IncompleteReviewPolicy::Block
14076 ));
14077 }
14078
14079 #[test]
14080 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14081 assert!(!round_is_clean(
14082 1,
14083 true,
14084 1,
14085 2,
14086 0,
14087 IncompleteReviewPolicy::Warn
14088 ));
14089 }
14090
14091 #[test]
14092 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14093 assert!(round_is_clean(
14094 0,
14095 true,
14096 1,
14097 2,
14098 0,
14099 IncompleteReviewPolicy::Warn
14100 ));
14101 }
14102
14103 #[test]
14104 fn a_full_panel_with_an_open_finding_is_not_clean() {
14105 assert!(!round_is_clean(
14106 1,
14107 true,
14108 2,
14109 2,
14110 0,
14111 IncompleteReviewPolicy::Block
14112 ));
14113 }
14114
14115 #[test]
14116 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14117 assert!(!round_is_clean(
14118 0,
14119 false,
14120 2,
14121 2,
14122 0,
14123 IncompleteReviewPolicy::Block
14124 ));
14125 }
14126
14127 #[test]
14134 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14135 assert!(round_is_clean(
14138 0,
14139 true,
14140 1,
14141 2,
14142 1,
14143 IncompleteReviewPolicy::Block
14144 ));
14145 }
14146
14147 #[test]
14148 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14149 assert!(!round_is_clean(
14152 0,
14153 true,
14154 1,
14155 2,
14156 0,
14157 IncompleteReviewPolicy::Block
14158 ));
14159 }
14160
14161 #[test]
14162 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14163 assert!(!round_is_clean(
14164 1,
14165 true,
14166 1,
14167 2,
14168 1,
14169 IncompleteReviewPolicy::Block
14170 ));
14171 assert!(!round_is_clean(
14172 0,
14173 false,
14174 1,
14175 2,
14176 1,
14177 IncompleteReviewPolicy::Block
14178 ));
14179 }
14180
14181 #[test]
14182 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14183 assert!(!round_is_clean(
14187 0,
14188 true,
14189 0,
14190 2,
14191 2,
14192 IncompleteReviewPolicy::Block
14193 ));
14194 }
14195
14196 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14197 CommandOutcome {
14198 command: "test".to_owned(),
14199 code,
14200 output_tail: String::new(),
14201 duration_ms: 0,
14202 resource_blocked,
14203 }
14204 }
14205
14206 #[cfg(unix)]
14207 #[tokio::test]
14208 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14209 ask_test_home();
14210 let temp = tempfile::tempdir().unwrap();
14211 let mut runner = runner_at(RunStatus::Gating);
14212 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14213 let commands = vec![
14214 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14215 "touch must-not-run".to_owned(),
14216 ];
14217 let (outcomes, retried, pids) = run_e2e_with_retry(
14218 &mut runner.state,
14219 &shell,
14220 &commands,
14221 temp.path(),
14222 Duration::from_secs(5),
14223 "fixture",
14224 )
14225 .await;
14226 assert_eq!(outcomes.len(), 1);
14227 assert!(outcomes[0].resource_blocked);
14228 assert!(!outcomes[0].build_failed());
14229 assert!(!retried);
14230 assert!(pids.is_empty());
14231 assert!(!temp.path().join("must-not-run").exists());
14232 assert_eq!(
14233 runner.state.resource_block.as_ref().unwrap().kind,
14234 "incremental cache admission"
14235 );
14236 }
14237
14238 #[test]
14239 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14240 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14241 assert!(
14242 !verify_inconclusive(&[outcome(Some(1), false)]),
14243 "an ordinary failure is still evidence about the patch"
14244 );
14245 assert!(verify_inconclusive(&[outcome(None, true)]));
14246 assert!(
14247 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14248 "one inconclusive outcome taints the whole batch"
14249 );
14250 assert!(!verify_inconclusive(&[]));
14251 }
14252
14253 #[tokio::test]
14254 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14255 let calls = std::sync::atomic::AtomicUsize::new(0);
14259 let started = Instant::now();
14260 wait_for_pids_with(
14261 &[123],
14262 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14263 Duration::from_millis(5),
14264 Duration::from_secs(5),
14265 )
14266 .await;
14267 assert!(
14268 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14269 "must keep checking rather than deciding on the first answer"
14270 );
14271 assert!(
14272 started.elapsed() < Duration::from_secs(1),
14273 "must return the moment it is confirmed dead, not wait out the ceiling"
14274 );
14275 }
14276
14277 #[tokio::test]
14278 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14279 let started = Instant::now();
14280 wait_for_pids_with(
14281 &[123],
14282 |_| true, Duration::from_millis(5),
14284 Duration::from_millis(30),
14285 )
14286 .await;
14287 let elapsed = started.elapsed();
14288 assert!(
14289 elapsed >= Duration::from_millis(30),
14290 "must not give up before its own ceiling: {elapsed:?}"
14291 );
14292 assert!(
14293 elapsed < Duration::from_secs(1),
14294 "must not wait past its own ceiling either: {elapsed:?}"
14295 );
14296 }
14297
14298 #[tokio::test]
14299 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14300 let started = Instant::now();
14301 wait_for_pids_with(
14302 &[],
14303 |_| true,
14304 Duration::from_secs(5),
14305 Duration::from_secs(5),
14306 )
14307 .await;
14308 assert!(
14309 started.elapsed() < Duration::from_millis(200),
14310 "an empty pid list has nothing to confirm"
14311 );
14312 }
14313
14314 fn review_round(
14320 clean: bool,
14321 blocking: usize,
14322 answered: usize,
14323 expected: usize,
14324 progressed: bool,
14325 e2e_ok: bool,
14326 ) -> ReviewRound {
14327 ReviewRound {
14328 round: 1,
14329 head: "h".to_owned(),
14330 verified_head: None,
14331 verified_at: None,
14332 reviews: Vec::new(),
14333 e2e: vec![CommandOutcome {
14334 command: "test".to_owned(),
14335 code: Some(if e2e_ok { 0 } else { 1 }),
14336 output_tail: String::new(),
14337 duration_ms: 0,
14338 resource_blocked: false,
14339 }],
14340 verify_retried: false,
14341 e2e_deferred: false,
14342 e2e_defer_reason: None,
14343 fix: None,
14344 blocking,
14345 answered,
14346 expected,
14347 clean,
14348 progressed,
14349 vote_split: false,
14350 reconsideration: Vec::new(),
14351 verdict: None,
14352 }
14353 }
14354
14355 #[test]
14356 fn review_conclusion_is_none_when_nothing_has_run() {
14357 assert_eq!(review_conclusion(&[], 3), None);
14358 }
14359
14360 #[test]
14361 fn review_conclusion_is_none_while_rounds_remain() {
14362 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14363 assert_eq!(review_conclusion(&rounds, 3), None);
14364 }
14365
14366 #[test]
14367 fn review_conclusion_is_gating_once_a_round_is_clean() {
14368 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14369 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14370 }
14371
14372 #[test]
14373 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14374 let rounds = vec![
14375 review_round(false, 1, 2, 2, true, true),
14376 review_round(false, 1, 2, 2, true, true),
14377 ];
14378 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14379 }
14380
14381 #[test]
14382 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14383 let rounds = vec![
14384 review_round(false, 1, 2, 2, true, true),
14385 review_round(false, 1, 2, 2, true, false),
14386 ];
14387 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14388 }
14389
14390 #[test]
14391 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14392 let mut blocked = review_round(false, 1, 2, 2, true, false);
14399 blocked.e2e[0].resource_blocked = true;
14400 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14401 assert_eq!(review_conclusion(&rounds, 2), None);
14402 }
14403
14404 #[test]
14405 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14406 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14408 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14409 }
14410
14411 #[test]
14412 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14413 let rounds = vec![
14414 review_round(false, 1, 2, 2, false, true),
14415 review_round(false, 1, 2, 2, false, true),
14416 ];
14417 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14418 }
14419
14420 fn secs(n: u64) -> Duration {
14421 Duration::from_secs(n)
14422 }
14423
14424 #[tokio::test]
14427 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14428 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14429 if std::env::var_os(CHILD).is_none() {
14430 let output = std::process::Command::new(std::env::current_exe().unwrap())
14434 .args([
14435 "--exact",
14436 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14437 "--nocapture",
14438 ])
14439 .env(CHILD, "1")
14440 .env_remove("MAGI_WORKER_HOME")
14441 .env_remove("MAGI_WORKER_ID")
14442 .output()
14443 .unwrap();
14444 assert!(
14445 output.status.success()
14446 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14447 "{}\n{}",
14448 String::from_utf8_lossy(&output.stdout),
14449 String::from_utf8_lossy(&output.stderr)
14450 );
14451 return;
14452 }
14453 ask_test_home();
14454 let tmp = tempfile::tempdir().unwrap();
14455 let repo = tmp.path().join("repo");
14456 std::fs::create_dir_all(&repo).unwrap();
14457 init_repo(&repo);
14458 let g = |args: &[&str]| {
14459 let out = std::process::Command::new("git")
14460 .args(args)
14461 .current_dir(&repo)
14462 .quiet()
14463 .output()
14464 .unwrap();
14465 assert!(
14466 out.status.success(),
14467 "{}",
14468 String::from_utf8_lossy(&out.stderr)
14469 );
14470 };
14471 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14472 g(&["checkout", "-b", "historical"]);
14473 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14474 g(&["add", "."]);
14475 g(&["commit", "-m", "old alternative"]);
14476 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14477 g(&["checkout", "-b", "magi/task-selected", &base]);
14478 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14479 g(&["add", "."]);
14480 g(&["commit", "-m", "measurement infrastructure"]);
14481 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14482 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14483 g(&["add", "."]);
14484 g(&["commit", "-m", "dependent implementation"]);
14485 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14486 g(&["checkout", "main"]);
14487 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14488 g(&["add", "."]);
14489 g(&["commit", "-m", "unrelated base change"]);
14490 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14491 let origin = tmp.path().join("origin.git");
14492 g(&[
14493 "clone",
14494 "--bare",
14495 repo.to_str().unwrap(),
14496 origin.to_str().unwrap(),
14497 ]);
14498 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14499 g(&["fetch", "origin"]);
14500 let selection = crate::pr_recovery::Selection {
14501 url: "https://example.test/project/pull/1".into(),
14502 branch: "magi/task-selected".into(),
14503 head: head.clone(),
14504 source_head: None,
14505 source_run: None,
14506 source_proof: None,
14507 disposition: Some(crate::pr_recovery::Disposition::Review),
14508 instruction: "preserve measured behavior".into(),
14509 approved_at: jiff::Timestamp::now(),
14510 };
14511 let mut template = runner_at(RunStatus::Reviewing);
14512 template.state.repo = repo.clone();
14513 template.state.base_commit = current.clone();
14514 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14515 template.state.origin = Some(crate::run::Origin::queue("selected"));
14516 template.state.pr_head = Some(selection.branch.clone());
14517 template.state.pr_recovery = Some(selection.clone());
14518 let mut runner =
14519 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14520 .await
14521 .unwrap();
14522 let wt = runner.state.winner().unwrap().worktree.clone();
14523 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14524 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14525 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14526 assert!(runner.state.seeds.is_empty());
14527 assert_eq!(runner.state.instruction, "preserve measured behavior");
14528 runner.sync_to_base().await.unwrap();
14529 assert_eq!(
14530 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14531 "prerequisite"
14532 );
14533 assert_eq!(
14534 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14535 "measured implementation"
14536 );
14537 assert_eq!(
14538 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14539 "current main change"
14540 );
14541 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14542 assert_eq!(
14543 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14544 head
14545 );
14546
14547 let mut partial = runner_at(RunStatus::Reviewing);
14549 partial.state.repo = repo.clone();
14550 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14551 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14552 run: Some("exit 1".into()),
14553 ..Default::default()
14554 }];
14555 let selection = runner.state.pr_recovery.clone().unwrap();
14556 partial.state.pr_recovery = Some(selection.clone());
14557 partial.state.pr_head = Some(selection.branch.clone());
14558 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14559 let id = partial.state.id.clone();
14560 let error = match Runner::open_selected(
14561 &repo,
14562 partial.state,
14563 partial.roles,
14564 1,
14565 runner.state.base_commit.clone(),
14566 selection,
14567 )
14568 .await
14569 {
14570 Ok(_) => panic!("setup must fail"),
14571 Err(e) => e,
14572 };
14573 assert_eq!(
14574 error
14575 .downcast_ref::<crate::run::StartupFailure>()
14576 .unwrap()
14577 .run,
14578 id
14579 );
14580 let failed = RunState::load(&id).unwrap();
14581 assert_eq!(failed.status, RunStatus::Failed);
14582 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14583 assert!(Runner::resume(&id).is_err());
14584 runner.state.status = RunStatus::Ready;
14585 runner.state.driver_exited = true;
14586 runner.state.save().unwrap();
14587 let mut retry = runner_at(RunStatus::Reviewing);
14588 retry.state.repo = repo.clone();
14589 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14590 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14591 reselected.approved_at = jiff::Timestamp::now();
14592 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14593 retry.state.pr_head = Some(reselected.branch.clone());
14594 retry.state.pr_recovery = Some(reselected.clone());
14595 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14596 .await
14597 .unwrap();
14598 let restarted = Runner::open_selected(
14599 &repo,
14600 retry.state,
14601 retry.roles,
14602 1,
14603 runner.state.base_commit.clone(),
14604 reselected,
14605 )
14606 .await
14607 .unwrap();
14608 assert_eq!(
14609 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14610 .await
14611 .unwrap(),
14612 head
14613 );
14614 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14615 assert_eq!(
14616 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14617 head
14618 );
14619 }
14620
14621 #[tokio::test]
14622 #[cfg(unix)]
14623 async fn fallback_and_interrupted_restoration_require_coordination() {
14624 use std::os::fd::AsRawFd as _;
14625 ask_test_home();
14626 let tmp = tempfile::tempdir().unwrap();
14627 let mut runner = registered_runner(tmp.path()).await;
14628 runner.state.config.graph.timeout_resource_wait = 1;
14629 let winner = runner.state.candidates[0].clone();
14630 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14631 let leftover = winner.worktree.join("leftover");
14632 std::fs::write(&leftover, "untrusted edits").unwrap();
14633 let next = tmp.path().join("next-fixer");
14634 let mut failed = spec("first");
14635 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14636 let mut successor = spec("second");
14637 successor.command = vec![
14638 "sh".into(),
14639 "-c".into(),
14640 format!("touch '{}'; echo done", next.display()),
14641 ];
14642 let attempts = vec![
14643 (failed.clone(), "first".into()),
14644 (successor.clone(), "second".into()),
14645 ];
14646 let lock_path = runner.state.repo.join(".git/updater.lock");
14647 let external = std::fs::File::create(&lock_path).unwrap();
14648 assert_eq!(
14649 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14650 0
14651 );
14652 let artifacts = tmp.path().join("artifacts");
14653 let build = |spec, seat| SeatJob {
14654 spec,
14655 seat,
14656 cwd: winner.worktree.clone(),
14657 prompt: "fixture".into(),
14658 timeout: Duration::from_secs(10),
14659 allow_write: true,
14660 sessions: false,
14661 artifacts: artifacts.clone(),
14662 stem: "fix".into(),
14663 handover: None,
14664 };
14665 let (_, _, out) = runner
14666 .ask_fixer_among(
14667 attempts.clone(),
14668 "graft",
14669 None,
14670 Some((&winner.worktree, &head)),
14671 build,
14672 )
14673 .await;
14674 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14675 assert!(leftover.exists(), "contention must prevent clean");
14676 assert!(!next.exists());
14677 assert!(runner.state.handovers.is_empty());
14678 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14680 from_run: "earlier".into(),
14681 generation: 1,
14682 mode: crate::recovery::Mode::VerifyRepair,
14683 reasons: Vec::new(),
14684 reviewed_head: None,
14685 rounds: Vec::new(),
14686 open_findings: Vec::new(),
14687 evidence: None,
14688 criteria_inherited: true,
14689 repair: crate::recovery::Repair::Started {
14690 before: head.clone(),
14691 },
14692 });
14693 assert!(
14694 runner
14695 .recovery_repair(&winner)
14696 .await
14697 .unwrap_err()
14698 .is::<crate::repo_lock::ResourceBlocked>()
14699 );
14700 assert!(matches!(
14701 runner.state.recovery.as_ref().unwrap().repair,
14702 crate::recovery::Repair::Started { .. }
14703 ));
14704 assert!(leftover.exists());
14705 drop(external);
14706 runner.recovery_repair(&winner).await.unwrap();
14707 assert!(!leftover.exists());
14708 assert!(matches!(
14709 runner.state.recovery.as_ref().unwrap().repair,
14710 crate::recovery::Repair::Done { .. }
14711 ));
14712 std::fs::remove_file(&lock_path).unwrap();
14714 std::fs::create_dir(&lock_path).unwrap();
14715 std::fs::write(&leftover, "untrusted edits").unwrap();
14716 let (_, _, out) = runner
14717 .ask_fixer_among(
14718 attempts,
14719 "graft",
14720 None,
14721 Some((&winner.worktree, &head)),
14722 build,
14723 )
14724 .await;
14725 assert!(matches!(out, AgentOutcome::Failed(_)));
14726 assert!(leftover.exists());
14727 assert!(!next.exists());
14728 assert_eq!(
14729 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14730 head
14731 );
14732 }
14733
14734 #[cfg(unix)]
14735 async fn registered_runner(root: &Path) -> Runner {
14736 repository_runner(root, crate::config::test_coordination()).await
14737 }
14738
14739 async fn repository_runner(
14740 root: &Path,
14741 coordination: Option<crate::config::Coordination>,
14742 ) -> Runner {
14743 ask_test_home();
14744 let repo = root.join("repo");
14745 std::fs::create_dir_all(&repo).unwrap();
14746 init_repo(&repo);
14747 if coordination.is_some() {
14748 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14749 }
14750 let mut runner = runner_at(RunStatus::Prep);
14751 runner.state.repo = repo.clone();
14752 runner.state.config.coordination = coordination;
14753 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14754 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14755 runner.state.config.graph.advise = false;
14756 runner.roles.implementers = vec![spec("implement")];
14757 runner.roles.judges = vec![spec("judge")];
14758 runner.prep().await.unwrap();
14759 runner
14760 }
14761
14762 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14763
14764 #[tokio::test]
14765 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14766 let tmp = tempfile::tempdir().unwrap();
14767 let mut runner = repository_runner(tmp.path(), None).await;
14768 let prompts = runner.state.config.prompts.clone();
14769 let run = runner.state.id.clone();
14770 let ctx = WaveCtx {
14771 carry_seats: false,
14772 pauses: None,
14773 run: &run,
14774 node: "advise",
14775 prompts: &prompts,
14776 cache: None,
14777 round: None,
14778 };
14779 let jobs: Vec<_> = ["alpha", "beta"]
14780 .into_iter()
14781 .map(|id| SeatJob {
14782 spec: scripted(id, AUTH_OUTAGE),
14783 seat: SeatState::new(id, id, 7),
14784 prompt: "propose".into(),
14785 cwd: runner.state.candidates[0].worktree.clone(),
14786 timeout: Duration::from_secs(10),
14787 allow_write: false,
14788 sessions: false,
14789 artifacts: agent::artifacts_dir(&runner.state.dir()),
14790 stem: id.into(),
14791 handover: None,
14792 })
14793 .collect();
14794 let mut losses = Vec::new();
14795 let results = ask_wave_with(
14796 jobs.clone(),
14797 runner.sem.clone(),
14798 2,
14799 &[],
14800 &ctx,
14801 &mut losses,
14802 &mut runner.state,
14803 &|_: &str| Ok(()),
14804 )
14805 .await;
14806 assert!(
14807 results
14808 .iter()
14809 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14810 );
14811 assert_eq!(runner.state.quota.len(), 2);
14812 assert!(
14813 runner
14814 .state
14815 .quota
14816 .iter()
14817 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14818 );
14819 assert!(!runner.state.advise_attempted);
14820 runner.stop_for_resource().unwrap();
14821 assert_eq!(runner.state.status, RunStatus::Blocked);
14822 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14823 assert_eq!(
14824 runner.state.resource_block.as_ref().unwrap().kind,
14825 "agent authentication"
14826 );
14827 runner.state.resource_block = None;
14830 runner.state.quota.clear();
14831 let mut mixed = jobs;
14832 mixed[1].spec.command = vec!["false".into()];
14833 let results = ask_wave_with(
14834 mixed,
14835 runner.sem.clone(),
14836 0,
14837 &[],
14838 &ctx,
14839 &mut losses,
14840 &mut runner.state,
14841 &|_: &str| Ok(()),
14842 )
14843 .await;
14844 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14845 assert!(runner.state.resource_block.is_none());
14846 assert_eq!(losses.len(), 1);
14847 }
14848
14849 #[tokio::test]
14850 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14851 let tmp = tempfile::tempdir().unwrap();
14852 let mut runner = repository_runner(tmp.path(), None).await;
14853 let implementer = scripted("implement", AUTH_OUTAGE);
14854 runner.roles.implementers = vec![implementer.clone()];
14855 runner.state.config.agents = vec![implementer];
14856 runner.implement().await.unwrap();
14857 assert_eq!(runner.state.status, RunStatus::Blocked);
14858 assert_eq!(runner.state.quota.len(), 1);
14859 assert_eq!(
14860 runner.state.quota[0].cause,
14861 agent::QuotaCause::Authentication
14862 );
14863 let candidate = &runner.state.candidates[0];
14864 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14865 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14866 }
14867
14868 #[tokio::test]
14869 #[cfg(unix)]
14870 async fn parked_implementer_persists_completed_work() {
14871 for registered in [false, true] {
14872 let tmp = tempfile::tempdir().unwrap();
14873 let mut runner = registered_runner(tmp.path()).await;
14874 if !registered {
14875 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14876 }
14877 let marker = tmp.path().join("started");
14878 let release = tmp.path().join("release");
14879 let mut implementer = spec("implement");
14880 implementer.command = vec![
14881 "sh".into(),
14882 "-c".into(),
14883 format!(
14884 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14885 marker.display(),
14886 release.display()
14887 ),
14888 ];
14889 runner.roles.implementers = vec![implementer.clone()];
14890 runner.state.config.agents = vec![implementer];
14891 let pause = runner.pause.clone();
14892 let (result, ()) = tokio::join!(runner.implement(), async {
14893 tokio::time::timeout(Duration::from_secs(10), async {
14894 while !marker.exists() {
14895 tokio::time::sleep(Duration::from_millis(10)).await;
14896 }
14897 })
14898 .await
14899 .unwrap();
14900 pause.park();
14901 std::fs::write(&release, "go").unwrap();
14902 });
14903 result.unwrap();
14904 let candidate = &runner.state.candidates[0];
14905 assert!(candidate.commits > 0, "{candidate:?}");
14906 assert!(candidate.summary.contains("saved-result"));
14907 assert_eq!(candidate.files, 1);
14908 let before = candidate.commits;
14909 runner.implement().await.unwrap();
14910 assert_eq!(runner.state.candidates[0].commits, before);
14911 }
14912 }
14913
14914 #[tokio::test]
14915 #[cfg(unix)]
14916 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14917 let tmp = tempfile::tempdir().unwrap();
14918 let mut runner = registered_runner(tmp.path()).await;
14919 let winner = runner.state.candidates[0].clone();
14920 let cache = tmp.path().join("cache");
14921 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14922 runner.state.config.graph.timeout_resource_wait = 0;
14923 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14924 let crate::cache::AcquireOutcome::Acquired(held) =
14925 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14926 else {
14927 panic!("cache must be free")
14928 };
14929 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14930 from_run: "earlier".into(),
14931 generation: 1,
14932 mode: crate::recovery::Mode::VerifyRepair,
14933 reasons: Vec::new(),
14934 reviewed_head: None,
14935 rounds: Vec::new(),
14936 open_findings: Vec::new(),
14937 evidence: None,
14938 criteria_inherited: true,
14939 repair: crate::recovery::Repair::NotStarted,
14940 });
14941 runner.recovery_repair(&winner).await.unwrap();
14942 assert!(matches!(
14943 runner.state.recovery.as_ref().unwrap().repair,
14944 crate::recovery::Repair::WaitingForRestore { .. }
14945 ));
14946 drop(held);
14947 runner.state.resource_block = None;
14948 runner.state.config.graph.timeout_resource_wait = 1;
14949 runner.recovery_repair(&winner).await.unwrap();
14950 assert!(matches!(
14951 runner.state.recovery.as_ref().unwrap().repair,
14952 crate::recovery::Repair::Done { .. }
14953 ));
14954 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14955 }
14956
14957 #[tokio::test]
14958 #[cfg(unix)]
14959 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14960 use std::os::fd::AsRawFd as _;
14961 for graft in [false, true] {
14962 let tmp = tempfile::tempdir().unwrap();
14963 let mut runner = registered_runner(tmp.path()).await;
14964 runner.state.config.graph.timeout_resource_wait = 1;
14965 let winner = runner.state.candidates[0].clone();
14966 let marker = tmp.path().join("fixer-started");
14967 let mut fixer = spec("fixer");
14968 fixer.command = vec![
14969 "sh".into(),
14970 "-c".into(),
14971 format!("echo started >> '{}'; echo done", marker.display()),
14972 ];
14973 runner.roles.fixer = Some(vec![fixer]);
14974 if graft {
14975 let mut loser = winner.clone();
14976 loser.label = 'B';
14977 loser.index = 1;
14978 runner.state.candidates.push(loser);
14979 runner.state.config.graph.graft_runner_up = true;
14980 runner.state.tally = Some(Tally {
14981 first_choice: BTreeMap::new(),
14982 borda: BTreeMap::new(),
14983 winner: winner.label,
14984 rankings: 1,
14985 unanimous_initial: true,
14986 deliberated: false,
14987 changed_votes: 0,
14988 unanimous_final: true,
14989 tie_break: None,
14990 judges: 1,
14991 present: 1,
14992 quorum: 1,
14993 met_quorum: true,
14994 uncontested: None,
14995 });
14996 runner.state.judgements.push(crate::run::Judgement {
14997 judge: 1,
14998 seat: "judge-1".into(),
14999 agent: "judge".into(),
15000 ranking: vec![winner.label, 'B'],
15001 reasons: BTreeMap::new(),
15002 confidence: None,
15003 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15004 order: vec![0, 1],
15005 failed: None,
15006 duration_ms: 0,
15007 });
15008 } else {
15009 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15010 from_run: "earlier".into(),
15011 generation: 1,
15012 mode: crate::recovery::Mode::VerifyRepair,
15013 reasons: Vec::new(),
15014 reviewed_head: None,
15015 rounds: Vec::new(),
15016 open_findings: Vec::new(),
15017 evidence: None,
15018 criteria_inherited: true,
15019 repair: crate::recovery::Repair::NotStarted,
15020 });
15021 }
15022 let reader = crate::repo_lock::SeatLease::for_checkout(
15023 &winner.worktree,
15024 false,
15025 crate::config::test_coordination().as_ref(),
15026 )
15027 .await
15028 .unwrap();
15029 let pause = runner.pause.clone();
15030 let (result, ()) = tokio::join!(
15031 async {
15032 if graft {
15033 runner.graft().await
15034 } else {
15035 runner.recovery_repair(&winner).await
15036 }
15037 },
15038 async {
15039 tokio::time::sleep(Duration::from_millis(100)).await;
15040 pause.park();
15041 }
15042 );
15043 result.unwrap();
15044 assert!(!marker.exists(), "a blocked seat must not spawn");
15045 runner.state = RunState::load(&runner.state.id).unwrap();
15047 if graft {
15048 assert_eq!(
15049 runner.state.graft.as_ref().unwrap().outcome,
15050 GraftOutcome::WaitingForRestore
15051 );
15052 } else {
15053 assert!(matches!(
15054 runner.state.recovery.as_ref().unwrap().repair,
15055 crate::recovery::Repair::WaitingForRestore { .. }
15056 ));
15057 }
15058 drop(reader);
15059 runner.pause = Pause::new();
15060 runner.state.resource_block = None;
15061 let external =
15063 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15064 assert_eq!(
15065 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15066 0
15067 );
15068 let leftover = winner.worktree.join("leftover");
15069 std::fs::write(&leftover, "earlier fallback edits").unwrap();
15070 let result = if graft {
15071 runner.graft().await
15072 } else {
15073 runner.recovery_repair(&winner).await
15074 };
15075 assert!(
15076 result
15077 .unwrap_err()
15078 .is::<crate::repo_lock::ResourceBlocked>()
15079 );
15080 assert!(leftover.exists());
15081 assert!(!marker.exists());
15082 drop(external);
15083 runner.state.resource_block = None;
15084 let result = if graft {
15085 runner.graft().await
15086 } else {
15087 runner.recovery_repair(&winner).await
15088 };
15089 result.unwrap();
15090 assert!(!leftover.exists(), "restore must precede retry");
15091 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15092 if graft {
15093 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15094 } else {
15095 assert!(matches!(
15096 runner.state.recovery.as_ref().unwrap().repair,
15097 crate::recovery::Repair::Done { .. }
15098 ));
15099 }
15100 }
15101 }
15102
15103 #[tokio::test]
15104 #[cfg(unix)]
15105 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15106 use std::os::fd::AsRawFd as _;
15107 let tmp = tempfile::tempdir().unwrap();
15108 let mut runner = registered_runner(tmp.path()).await;
15109 runner.state.config.graph.timeout_resource_wait = 1;
15110 let winner = runner.state.candidates[0].clone();
15111 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15112 runner.state.tally = Some(Tally {
15113 first_choice: BTreeMap::new(),
15114 borda: BTreeMap::new(),
15115 winner: winner.label,
15116 rankings: 0,
15117 unanimous_initial: false,
15118 deliberated: false,
15119 changed_votes: 0,
15120 unanimous_final: false,
15121 tie_break: None,
15122 judges: 0,
15123 present: 0,
15124 quorum: 0,
15125 met_quorum: true,
15126 uncontested: Some("fixture".into()),
15127 });
15128 runner.state.graft = Some(GraftRecord {
15129 pre: pre.clone(),
15130 post: None,
15131 lines: None,
15132 limit: None,
15133 agent: None,
15134 outcome: GraftOutcome::Running,
15135 requested: Vec::new(),
15136 adoption: None,
15137 adoption_error: None,
15138 });
15139 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15140 git::commit_all(&winner.worktree, "unfinished graft")
15141 .await
15142 .unwrap();
15143 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15144 let leftover = winner.worktree.join("leftover");
15145 std::fs::write(&leftover, "untrusted edits").unwrap();
15146 let path = runner.state.repo.join(".git/updater.lock");
15147 let external = std::fs::File::create(&path).unwrap();
15148 assert_eq!(
15149 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15150 0
15151 );
15152 assert!(
15153 runner
15154 .graft()
15155 .await
15156 .unwrap_err()
15157 .is::<crate::repo_lock::ResourceBlocked>()
15158 );
15159 assert!(matches!(
15160 runner.state.graft.as_ref().unwrap().outcome,
15161 GraftOutcome::Running
15162 ));
15163 assert_eq!(
15164 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15165 unfinished
15166 );
15167 assert!(leftover.exists());
15168 drop(external);
15169 std::fs::remove_file(&path).unwrap();
15170 std::fs::create_dir(&path).unwrap();
15171 assert!(runner.graft().await.is_err());
15172 assert!(matches!(
15173 runner.state.graft.as_ref().unwrap().outcome,
15174 GraftOutcome::Running
15175 ));
15176 assert_eq!(
15177 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15178 unfinished
15179 );
15180 assert!(leftover.exists());
15181 std::fs::remove_dir(&path).unwrap();
15182 runner.graft().await.unwrap();
15183 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15184 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15185 assert!(!leftover.exists());
15186 }
15187
15188 #[tokio::test]
15189 #[cfg(unix)]
15190 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15191 use std::os::fd::AsRawFd as _;
15192 let tmp = tempfile::tempdir().unwrap();
15193 let mut runner = registered_runner(tmp.path()).await;
15194 runner.state.prep_complete = Some(false);
15195 runner.state.config.graph.timeout_resource_wait = 1;
15196 runner.state.save().unwrap();
15197 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15198 assert_eq!(
15199 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15200 0
15201 );
15202 runner.execute().await.unwrap();
15203 let disk = RunState::load(&runner.state.id).unwrap();
15204 assert!(disk.driver_exited);
15205 assert_eq!(disk.driver_pid, None);
15206 assert_eq!(runner.state.status, RunStatus::Blocked);
15207 assert_eq!(
15208 runner.state.resource_block.as_ref().unwrap().kind,
15209 "repository coordination"
15210 );
15211 assert!(runner.state.prep_failure.is_none());
15212 assert!(runner.state.seats.is_empty());
15213 runner.pause.park();
15214 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15217 .await
15218 .unwrap()
15219 .unwrap();
15220 assert!(runner.state.parked);
15221 assert!(runner.state.prep_failure.is_none());
15222 let disk = RunState::load(&runner.state.id).unwrap();
15223 assert!(disk.driver_exited);
15224 assert_eq!(disk.driver_pid, None);
15225 }
15226
15227 #[tokio::test]
15228 #[cfg(unix)]
15229 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15230 use std::os::fd::AsRawFd as _;
15231 let tmp = tempfile::tempdir().unwrap();
15232 let mut runner = registered_runner(tmp.path()).await;
15233 runner.state.prep_complete = Some(false);
15234 runner.state.config.graph.timeout_resource_wait = 30;
15235 runner.state.save().unwrap();
15236 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15237 assert_eq!(
15238 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15239 0
15240 );
15241 assert!(
15242 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15243 .await
15244 .is_err()
15245 );
15246 let disk = RunState::load(&runner.state.id).unwrap();
15247 assert!(disk.driver_exited);
15248 assert_eq!(disk.driver_pid, None);
15249 assert!(disk.status.resumable());
15250 assert!(runner.state.driver_exited);
15251 assert_eq!(runner.state.driver_pid, None);
15252 runner.state.save().unwrap();
15253 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15254 }
15255
15256 #[tokio::test]
15257 #[cfg(unix)]
15258 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15259 let tmp = tempfile::tempdir().unwrap();
15260 let mut runner = registered_runner(tmp.path()).await;
15261 let winner = runner.state.candidates[0].clone();
15262 let cache = tmp.path().join("cache");
15263 let started = tmp.path().join("pre-gate-started");
15264 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15265 runner.state.config.verify.gate =
15266 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15267 let home = crate::run::home();
15268 let owner =
15269 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15270 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15271 crate::cache::AcquireOutcome::Acquired(g) => g,
15272 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15273 };
15274 let mut gate = Box::pin(runner.run_gate(&winner));
15275 assert!(
15276 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15277 .await
15278 .is_err()
15279 );
15280 assert!(!started.exists(), "pre-gate must wait for the cache too");
15281 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15285 let editor = crate::repo_lock::SeatLease::checkout_with(
15286 &winner.worktree,
15287 true,
15288 crate::config::test_coordination().as_ref(),
15289 &wait,
15290 )
15291 .await
15292 .unwrap();
15293 drop(editor);
15294 guard.release();
15295 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15296 .await
15297 .unwrap()
15298 .unwrap();
15299 drop(gate);
15300 assert!(outcomes.iter().all(CommandOutcome::ok));
15301 assert!(started.exists());
15302 }
15303
15304 #[tokio::test]
15305 #[cfg(unix)]
15306 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15307 use std::os::fd::AsRawFd as _;
15308
15309 ask_test_home();
15310 let tmp = tempfile::tempdir().unwrap();
15311 let repo = tmp.path().join("repo");
15312 std::fs::create_dir_all(&repo).unwrap();
15313 init_repo(&repo);
15314 let common = repo.join(".git");
15315 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15316 let mut runner = runner_at(RunStatus::Prep);
15317 runner.state.repo = repo.clone();
15318 runner.state.config.coordination = crate::config::test_coordination();
15319 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15320 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15321 runner.state.config.graph.advise = false;
15322 runner.roles.implementers = vec![spec("implement")];
15323 runner.roles.judges = vec![spec("judge")];
15324 runner.prep().await.unwrap();
15325 let winner = runner.state.candidates[0].clone();
15326 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15329 let completed = tmp.path().join("seat-completed");
15330 runner.roles.implementers[0].command = vec![
15331 "sh".into(),
15332 "-c".into(),
15333 format!("touch '{}'; echo completed", completed.display()),
15334 ];
15335 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15336 assert_eq!(
15337 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15338 0
15339 );
15340 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15341 let mut implement = Box::pin(runner.implement());
15342 tokio::select! {
15343 result = &mut implement => panic!("rescue must wait: {result:?}"),
15344 result = tokio::time::timeout(Duration::from_secs(10), async {
15345 while !completed.exists() {
15346 tokio::time::sleep(Duration::from_millis(20)).await;
15347 }
15348 }) => result.unwrap(),
15349 }
15350 assert!(
15351 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15352 .await
15353 .is_err()
15354 );
15355 assert_eq!(
15356 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15357 before
15358 );
15359 assert_eq!(
15360 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15361 0
15362 );
15363 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15364 .await
15365 .unwrap()
15366 .unwrap();
15367 drop(implement);
15368 assert_ne!(
15369 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15370 before
15371 );
15372 assert!(git::is_clean(&winner.worktree).await.unwrap());
15373
15374 let started = tmp.path().join("pre-gate-started");
15375 let proceed = tmp.path().join("pre-gate-proceed");
15376 let checked = tmp.path().join("gate-ran");
15377 runner.state.config.verify.pre_gate = vec![format!(
15378 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15379 started.display(),
15380 proceed.display()
15381 )];
15382 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15383 assert_eq!(
15384 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15385 0
15386 );
15387 let mut gate = Box::pin(runner.run_gate(&winner));
15388 assert!(
15389 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15390 .await
15391 .is_err()
15392 );
15393 assert!(!started.exists());
15394 assert!(!checked.exists());
15395 assert!(git::is_clean(&winner.worktree).await.unwrap());
15396 assert_eq!(
15397 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15398 0
15399 );
15400 tokio::select! {
15401 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15402 result = tokio::time::timeout(Duration::from_secs(10), async {
15403 while !started.exists() {
15404 tokio::time::sleep(Duration::from_millis(20)).await;
15405 }
15406 }) => result.unwrap(),
15407 }
15408 assert_eq!(
15409 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15410 -1,
15411 "mechanical edits must hold the real lock"
15412 );
15413 std::fs::write(proceed, "ready").unwrap();
15414 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15415 .await
15416 .unwrap()
15417 .unwrap();
15418 drop(gate);
15419 assert!(outcomes.iter().all(CommandOutcome::ok));
15420 assert!(checked.exists());
15421 assert!(runner.state.pre_gate_commit.is_some());
15422 assert!(git::is_clean(&winner.worktree).await.unwrap());
15423 assert_eq!(
15424 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15425 0,
15426 "gate completion must release coordination"
15427 );
15428 drop(outsider);
15429 std::fs::remove_file(common.join("updater.lock")).unwrap();
15430 std::fs::create_dir(common.join("updater.lock")).unwrap();
15431 std::fs::remove_file(&started).unwrap();
15432 std::fs::remove_file(&checked).unwrap();
15433 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15434 let error = runner.run_gate(&winner).await.unwrap_err();
15435 assert!(
15436 error
15437 .to_string()
15438 .contains("acquire mandatory repository lock")
15439 );
15440 assert!(
15441 !started.exists(),
15442 "failed acquisition must prevent pre-gate"
15443 );
15444 assert!(!checked.exists(), "failed acquisition must prevent gate");
15445 assert!(git::is_clean(&winner.worktree).await.unwrap());
15446 assert_eq!(
15447 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15448 head
15449 );
15450 }
15451
15452 #[tokio::test]
15453 #[cfg(unix)]
15454 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15455 use std::os::fd::AsRawFd as _;
15456 use std::time::Duration;
15457
15458 ask_test_home();
15459 let tmp = tempfile::tempdir().unwrap();
15460 let repo = tmp.path().join("repo");
15461 std::fs::create_dir_all(&repo).unwrap();
15462 init_repo(&repo);
15463 let common = repo.join(".git");
15464 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15465 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15466 assert_eq!(
15467 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15468 0
15469 );
15470 let mut runner = runner_at(RunStatus::Prep);
15471 runner.state.repo = repo.clone();
15472 runner.state.config.coordination = crate::config::test_coordination();
15473 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15474 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15475 runner.state.config.graph.advise = true;
15476 runner.state.config.graph.advisors = 1;
15477 runner.roles.implementers = vec![spec("implement")];
15478 runner.roles.judges = vec![spec("judge")];
15479 let root = runner.state.worktree_root();
15480 let started = tmp.path().join("setup-started");
15481 let proceed = tmp.path().join("setup-proceed");
15482 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15483 run: Some(format!(
15484 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15485 started.display(),
15486 proceed.display()
15487 )),
15488 ..Default::default()
15489 }];
15490 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15491 let config = git::git(&repo, &["config", "--local", "--list"])
15492 .await
15493 .unwrap();
15494 let mut prep = Box::pin(runner.prep());
15495 assert!(
15496 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15497 .await
15498 .is_err()
15499 );
15500 assert!(
15501 !root.exists(),
15502 "contention must prevent even checkout creation"
15503 );
15504 assert!(!started.exists(), "contention must prevent setup");
15505 assert_eq!(
15506 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15507 branches
15508 );
15509 assert_eq!(
15510 git::git(&repo, &["config", "--local", "--list"])
15511 .await
15512 .unwrap(),
15513 config
15514 );
15515 drop(lock);
15516 tokio::select! {
15517 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15518 result = tokio::time::timeout(Duration::from_secs(10), async {
15519 while !started.exists() {
15520 tokio::time::sleep(Duration::from_millis(20)).await;
15521 }
15522 }) => result.unwrap(),
15523 }
15524 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15525 assert_eq!(
15526 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15527 -1
15528 );
15529 assert!(
15530 tokio::time::timeout(
15531 Duration::from_millis(100),
15532 crate::repo_lock::SeatLease::for_checkout(
15533 &repo,
15534 true,
15535 crate::config::test_coordination().as_ref()
15536 )
15537 )
15538 .await
15539 .is_err(),
15540 "preparation must also exclude local editors"
15541 );
15542 std::fs::write(proceed, "ready").unwrap();
15543 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15544 .await
15545 .unwrap()
15546 .unwrap();
15547 drop(prep);
15548 assert_eq!(runner.state.prep_complete, Some(true));
15549 assert!(root.join("judge-1").exists());
15550 assert!(root.join("advisor-1").exists());
15551 assert_eq!(
15552 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15553 0
15554 );
15555 }
15556
15557 #[tokio::test]
15558 #[cfg(unix)]
15559 async fn preparation_lock_failure_prevents_repository_changes() {
15560 ask_test_home();
15561 let tmp = tempfile::tempdir().unwrap();
15562 let repo = tmp.path().join("repo");
15563 std::fs::create_dir_all(&repo).unwrap();
15564 init_repo(&repo);
15565 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15566 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15569 let mut runner = runner_at(RunStatus::Prep);
15570 runner.state.repo = repo.clone();
15571 runner.state.config.coordination = crate::config::test_coordination();
15572 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15573 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15574 runner.roles.implementers = vec![spec("implement")];
15575 let error = runner.prep().await.unwrap_err();
15576 assert!(
15577 error
15578 .to_string()
15579 .contains("acquire mandatory repository lock")
15580 );
15581 assert!(!runner.state.worktree_root().exists());
15582 assert!(runner.state.candidates.is_empty());
15583 assert!(!repo.join(".git/config.worktree").exists());
15584 assert_eq!(
15585 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15586 "* main"
15587 );
15588 }
15589
15590 #[tokio::test]
15591 async fn preparation_records_creation_intent_before_git_can_fail() {
15592 ask_test_home();
15593 let tmp = tempfile::tempdir().unwrap();
15594 let repo = tmp.path().join("repo");
15595 std::fs::create_dir_all(&repo).unwrap();
15596 init_repo(&repo);
15597 let mut runner = runner_at(RunStatus::Prep);
15598 runner.state.repo = repo.clone();
15599 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15600 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15601 runner.state.config.blind.commit_msg_hook = false;
15602 runner.roles.implementers = vec![spec("one")];
15603 let label = blind::assign_labels(1, runner.state.seed)[0];
15604 let branch = runner.state.branch_for(label);
15605 git::git(&repo, &["branch", &branch]).await.unwrap();
15608 assert!(runner.prep().await.is_err());
15609 let saved = RunState::load(&runner.state.id).unwrap();
15610 assert!(saved.candidates.is_empty());
15611 let event = saved
15612 .events
15613 .iter()
15614 .find(|e| e.node == "prep-intent")
15615 .unwrap();
15616 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15617 assert_eq!(intent["branch"], branch);
15618 assert_eq!(intent["start"], runner.state.base_commit);
15619 assert_eq!(
15620 intent["worktree"],
15621 runner
15622 .state
15623 .worktree_root()
15624 .join(format!("cand-{label}"))
15625 .to_string_lossy()
15626 .as_ref()
15627 );
15628 assert_eq!(intent["picks"], serde_json::json!([]));
15629 }
15630
15631 #[tokio::test]
15632 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15633 ask_test_home();
15634 let tmp = tempfile::tempdir().unwrap();
15635 let repo = tmp.path().join("repo");
15636 std::fs::create_dir_all(&repo).unwrap();
15637 init_repo(&repo);
15638 let mut runner = runner_at(RunStatus::Prep);
15639 runner.state.repo = repo.clone();
15640 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15641 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15642 runner.state.sources.pick_commits.push(refs::Input {
15643 token: "missing".into(),
15644 sha: "0".repeat(40),
15645 provenance: refs::Provenance::ExplicitCli,
15646 });
15647 assert!(runner.prep().await.is_err());
15648 assert!(runner.state.candidates.is_empty());
15649 assert!(!runner.state.worktree_root().exists());
15650 }
15651
15652 #[tokio::test]
15653 async fn prose_and_legacy_inference_never_import_into_candidates() {
15654 ask_test_home();
15655 let tmp = tempfile::tempdir().unwrap();
15656 let repo = tmp.path().join("repo");
15657 std::fs::create_dir_all(&repo).unwrap();
15658 init_repo(&repo);
15659 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15660 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15661 .await
15662 .unwrap();
15663 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15664 git::git(&repo, &["add", "."]).await.unwrap();
15665 git::git(&repo, &["commit", "-m", "historical implementation"])
15666 .await
15667 .unwrap();
15668 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15669 git::git(&repo, &["checkout", "main"]).await.unwrap();
15670 for instruction in [
15671 format!("Example: magi/history/A at {source}"),
15672 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15673 format!("Never import magi/history/A or {source}"),
15674 format!("Conversation answer: I mentioned {source} only as background"),
15675 ] {
15676 let mut runner = runner_at(RunStatus::Prep);
15677 runner.state.repo = repo.clone();
15678 runner.state.base_commit = base.clone();
15679 runner.state.instruction = instruction.clone();
15680 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15681 runner.state.config.blind.commit_msg_hook = false;
15682 runner.state.seeds = vec![refs::Seed {
15683 token: "magi/history/A".into(),
15684 sha: source.clone(),
15685 kind: refs::SeedKind::Unmerged,
15686 branch: true,
15687 detail: String::new(),
15688 }];
15689 runner.roles.implementers = vec![spec("one")];
15690 assert_eq!(seeded_instruction(&runner.state), instruction);
15691 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15693 legacy.as_object_mut().unwrap().remove("sources");
15694 runner.state = serde_json::from_value(legacy).unwrap();
15695 assert_eq!(runner.state.sources, refs::Sources::default());
15696 runner.state.save().unwrap();
15697 runner.state = RunState::load(&runner.state.id).unwrap();
15698 runner.prep().await.unwrap();
15699 for candidate in &runner.state.candidates {
15700 assert_eq!(
15701 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15702 base
15703 );
15704 assert!(!candidate.worktree.join("unwanted").exists());
15705 }
15706 }
15707 }
15708
15709 #[tokio::test]
15710 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15711 ask_test_home();
15712 let tmp = tempfile::tempdir().unwrap();
15713 let repo = tmp.path().join("repo");
15714 std::fs::create_dir_all(&repo).unwrap();
15715 init_repo(&repo);
15716 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15717 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15718 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15719 git::git(&repo, &["add", "."]).await.unwrap();
15720 git::git(&repo, &["commit", "-m", "required seed"])
15721 .await
15722 .unwrap();
15723 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15724 git::git(&repo, &["checkout", "main"]).await.unwrap();
15725 let mut runner = runner_at(RunStatus::Prep);
15726 runner.state.repo = repo.clone();
15727 runner.state.base_commit = base.clone();
15728 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15729 runner.state.config.graph.advise = false;
15730 runner.state.config.blind.commit_msg_hook = false;
15731 runner.state.sources =
15732 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15733 .await
15734 .unwrap();
15735 runner.roles.implementers = vec![spec("one"), spec("two")];
15736 runner.roles.judges = vec![spec("judge")];
15737 let label = blind::assign_labels(2, runner.state.seed)[0];
15738 let branch = runner.state.branch_for(label);
15739 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15740 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15741 .await
15742 .unwrap();
15743 runner.state.candidates.push(Candidate {
15744 index: 0,
15745 label,
15746 agent: "one".into(),
15747 branch,
15748 worktree,
15749 summary: String::new(),
15750 stat: String::new(),
15751 files: 0,
15752 commits: 0,
15753 empty: false,
15754 failed: None,
15755 verified_noop: None,
15756 duration_ms: 0,
15757 folded: false,
15758 });
15759 runner.state.save().unwrap();
15760 runner.state = RunState::load(&runner.state.id).unwrap();
15762 assert_eq!(runner.state.prep_complete, Some(false));
15763 runner.prep().await.unwrap();
15764 assert_eq!(runner.state.prep_complete, Some(true));
15765 assert_eq!(runner.state.candidates.len(), 2);
15766 for candidate in &runner.state.candidates {
15767 assert_eq!(
15768 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15769 git::tree_of(&repo, &seed).await.unwrap()
15770 );
15771 assert_eq!(
15772 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15773 "required commit"
15774 );
15775 }
15776 assert!(runner.state.worktree_root().join("judge-1").exists());
15777 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15778 runner.prep().await.unwrap();
15779 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15780
15781 runner.state.prep_complete = Some(false);
15782 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15783 run: Some(format!(
15784 "test -f '{}'",
15785 tmp.path().join("dependency-ready").display()
15786 )),
15787 ..Default::default()
15788 }];
15789 assert!(runner.prep().await.is_err());
15790 assert!(runner.state.prep_failure.is_some());
15791 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15792 runner.state = RunState::load(&runner.state.id).unwrap();
15793 assert!(
15794 runner.prep().await.is_err(),
15795 "correcting the cause alone must not auto-retry"
15796 );
15797 let failed_at = runner.state.events.last().unwrap().at;
15798 assert!(
15799 runner.retry_preparation(failed_at).is_err(),
15800 "old approvals cannot unlock the hold"
15801 );
15802 let mut task = crate::queue::Task::new(
15803 "retry".into(),
15804 "retry setup".into(),
15805 repo.clone(),
15806 crate::queue::Source::Human,
15807 );
15808 task.start(runner.state.id.clone());
15809 task.hold_machine(Some("setup failed".into()));
15810 task.release_operator("cli", None);
15811 runner
15812 .retry_preparation(task.resumption.as_ref().unwrap().at)
15813 .unwrap();
15814 runner.prep().await.unwrap();
15815 assert!(runner.state.prep_failure.is_none());
15816 assert_eq!(runner.state.prep_complete, Some(true));
15817 assert_eq!(runner.state.candidates.len(), 2);
15818 assert!(runner.state.worktree_root().join("judge-1").exists());
15819 for candidate in &runner.state.candidates {
15820 assert_eq!(
15821 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15822 git::tree_of(&repo, &seed).await.unwrap()
15823 );
15824 }
15825 }
15826
15827 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15828 let mut heads = Vec::new();
15829 for candidate in candidates {
15830 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15831 }
15832 heads
15833 }
15834
15835 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15836 ask_test_home();
15837 let tmp = tempfile::tempdir().unwrap();
15838 let repo = tmp.path().join("repo");
15839 std::fs::create_dir(&repo).unwrap();
15840 init_repo(&repo);
15841 let mut runner = runner_at(RunStatus::Reviewing);
15842 runner.state.repo = repo.clone();
15843 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15844 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15845 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15846 let branch = format!("candidate-{label}");
15847 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15848 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15849 .await
15850 .unwrap();
15851 if label == 'B' {
15852 std::fs::write(worktree.join("change"), "winner").unwrap();
15853 git::rescue_commit(&worktree, "candidate change")
15854 .await
15855 .unwrap();
15856 }
15857 runner.state.candidates.push(Candidate {
15858 index,
15859 label,
15860 agent: "alpha".into(),
15861 branch,
15862 worktree,
15863 summary: "completed reply".into(),
15864 stat: String::new(),
15865 files: usize::from(label == 'B'),
15866 commits: usize::from(label == 'B'),
15867 empty: label == 'A',
15868 failed: None,
15869 verified_noop: None,
15870 duration_ms: 1,
15871 folded: false,
15872 });
15873 }
15874 runner.state.judge_skipped = true;
15875 runner.state.tally = Some(Tally {
15876 first_choice: BTreeMap::new(),
15877 borda: BTreeMap::new(),
15878 winner: 'B',
15879 rankings: 0,
15880 unanimous_initial: false,
15881 deliberated: false,
15882 changed_votes: 0,
15883 unanimous_final: false,
15884 tie_break: None,
15885 judges: 0,
15886 present: 0,
15887 quorum: 0,
15888 met_quorum: true,
15889 uncontested: Some("only B produced a change".into()),
15890 });
15891 runner.state.save().unwrap();
15892 (tmp, runner)
15893 }
15894
15895 #[tokio::test]
15896 #[cfg(unix)]
15897 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15898 for (solo, failed_checkpoint) in
15899 [(true, false), (false, false), (true, true), (false, true)]
15900 {
15901 let (_tmp, mut runner) = preservation_resume_fixture().await;
15902 if solo {
15903 runner.state.candidates.retain(|c| c.label == 'A');
15904 }
15905 runner.state.candidates[0].empty = false;
15906 runner.state.candidates[0].summary.clear();
15907 runner.state.tally = None;
15908 runner.state.judge_skipped = false;
15909 runner.state.status = RunStatus::Implementing;
15910 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15911 let mut implementer = spec("alpha");
15912 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15913 runner.roles.implementers = vec![implementer];
15914 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15915 if failed_checkpoint {
15916 std::fs::create_dir_all(&patch).unwrap();
15917 }
15918 runner.state.save().unwrap();
15919 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15920 drop(pressure);
15921 runner.state = RunState::load(&runner.state.id).unwrap();
15922 assert_eq!(runner.state.status, RunStatus::Blocked);
15923 assert!(runner.state.interrupted_implementations.contains(&'A'));
15924 assert_eq!(runner.state.candidates[0].commits, 0);
15925 assert!(!runner.state.candidates[0].empty);
15926 assert!(runner.state.candidates[0].failed.is_none());
15927 if failed_checkpoint {
15928 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15929 std::fs::remove_dir(patch).unwrap();
15930 }
15931 runner.state.resource_block = None;
15932 runner.state.config.disk.min_free_bytes = 0;
15933 runner.roles.implementers[0].command = vec![
15934 "sh".into(),
15935 "-c".into(),
15936 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15937 ];
15938 runner.implement().await.unwrap();
15939 assert!(runner.state.interrupted_implementations.is_empty());
15940 assert!(runner.state.candidate_preservation.is_empty());
15941 assert_eq!(runner.state.candidates[0].commits, 1);
15942 assert!(!runner.state.candidates[0].empty);
15943 assert!(
15944 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15945 .unwrap()
15946 .contains("retried empty interrupted seat")
15947 );
15948 if !solo {
15949 assert_eq!(runner.state.candidates[1].commits, 1);
15950 }
15951 }
15952 }
15953
15954 #[tokio::test]
15955 #[cfg(unix)]
15956 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15957 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15958 let (_tmp, mut runner) = preservation_resume_fixture().await;
15959 runner.state.candidates.retain(|c| c.label == 'A');
15960 runner.state.candidates[0].empty = false;
15961 runner.state.candidates[0].summary.clear();
15962 runner.state.tally = None;
15963 runner.state.judge_skipped = false;
15964 runner.state.status = RunStatus::Implementing;
15965 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15968 let wt = runner.state.candidates[0].worktree.clone();
15969 std::fs::write(
15970 wt.join("interrupted-source"),
15971 "retain implementation work\n",
15972 )
15973 .unwrap();
15974 if committed {
15975 git::rescue_commit(&wt, "implementation checkpoint")
15976 .await
15977 .unwrap();
15978 }
15979 let lock = git::git(
15980 &wt,
15981 &[
15982 "rev-parse",
15983 "--path-format=absolute",
15984 "--git-path",
15985 "index.lock",
15986 ],
15987 )
15988 .await
15989 .unwrap();
15990 if block_rescue {
15991 std::fs::write(&lock, "fixture-owned lock").unwrap();
15992 }
15993 let mut implementer = spec("alpha");
15994 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15995 runner.roles.implementers = vec![implementer];
15996 runner.state.save().unwrap();
15997 let result = runner.implement().await;
15998 drop(pressure);
15999 assert_eq!(result.is_err(), block_rescue);
16000 assert_eq!(
16001 runner.state.resource_block.as_ref().unwrap().kind,
16002 "disk pressure"
16003 );
16004 assert!(
16005 runner
16006 .state
16007 .events
16008 .iter()
16009 .any(|e| e.message.contains("agent execution interrupted"))
16010 );
16011 assert!(
16012 runner
16013 .state
16014 .dir()
16015 .join("artifacts/candidate-A-recovery/staged.patch")
16016 .exists()
16017 );
16018 if block_rescue {
16019 let saved = RunState::load(&runner.state.id).unwrap();
16020 assert!(saved.candidate_preservation.contains_key(&'A'));
16021 assert!(!saved.candidates[0].empty);
16022 assert_eq!(
16023 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16024 "retain implementation work\n"
16025 );
16026 std::fs::remove_file(lock).unwrap();
16027 runner.state = saved;
16028 } else {
16029 assert_eq!(runner.state.status, RunStatus::Blocked);
16030 assert_eq!(runner.state.candidates[0].commits, 1);
16031 }
16032 runner.roles.implementers.clear();
16035 runner.state.config.disk.min_free_bytes = 0;
16036 runner.state.resource_block = None;
16037 runner.implement().await.unwrap();
16038 assert!(runner.state.candidate_preservation.is_empty());
16039 assert_eq!(runner.state.candidates[0].commits, 1);
16040 assert!(!runner.state.candidates[0].empty);
16041 assert!(git::is_clean(&wt).await.unwrap());
16042 assert!(
16043 crate::run::read_artifact(&runner.state, "cand-A.patch")
16044 .unwrap()
16045 .contains("retain implementation work")
16046 );
16047 }
16048 }
16049
16050 #[tokio::test]
16051 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16052 let (_tmp, mut runner) = preservation_resume_fixture().await;
16053 runner.implement().await.unwrap();
16055 assert!(runner.state.candidate_preservation.is_empty());
16056 assert!(
16057 !runner
16058 .state
16059 .dir()
16060 .join("artifacts/candidate-A-recovery")
16061 .exists()
16062 );
16063 runner.fold_losers().await.unwrap();
16064 assert!(!runner.state.candidates[0].worktree.exists());
16065 for schema in [14, crate::run::SCHEMA] {
16066 runner.state.schema = schema;
16067 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16068 runner.state.save().unwrap();
16069 runner.state = RunState::load(&runner.state.id).unwrap();
16070 runner.implement().await.unwrap();
16071 assert!(runner.state.candidate_preservation.is_empty());
16072 assert!(runner.state.candidates[0].folded);
16073 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16074 }
16075 }
16076
16077 #[tokio::test]
16078 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16079 let (_tmp, mut runner) = preservation_resume_fixture().await;
16080 runner.state.candidates.retain(|c| c.label == 'A');
16081 runner.state.tally = None;
16082 runner.state.judge_skipped = false;
16083 let candidate = &mut runner.state.candidates[0];
16084 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16085 candidate.failed = Some("deployment validation unavailable".into());
16086 std::fs::write(
16087 candidate.worktree.join("repair"),
16088 "future launcher policy\n",
16089 )
16090 .unwrap();
16091 runner
16092 .state
16093 .candidate_preservation
16094 .insert('A', "awaiting persistence".into());
16095 runner.persist_candidates().await.unwrap();
16096 let candidate = &runner.state.candidates[0];
16097 assert!(!candidate.empty);
16098 assert_eq!(candidate.commits, 1);
16099 assert_eq!(candidate.files, 1);
16100 assert!(candidate.failed.is_none());
16101 assert!(candidate.summary.contains("EPERM"));
16102 assert!(
16103 crate::run::read_artifact(&runner.state, "cand-A.patch")
16104 .unwrap()
16105 .contains("future launcher policy")
16106 );
16107 assert!(runner.state.candidate_preservation.is_empty());
16108 }
16109
16110 #[tokio::test]
16111 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16112 let (_tmp, mut runner) = preservation_resume_fixture().await;
16113 let wt = runner.state.candidates[0].worktree.clone();
16114 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16115 runner.state.schema = 14;
16116 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16117 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16118 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16119 for _ in 0..2 {
16120 assert!(runner.implement().await.is_err());
16121 assert_eq!(runner.state.status, RunStatus::Blocked);
16122 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16123 assert!(
16124 runner.state.candidates[0].empty,
16125 "old verdict must remain unusable"
16126 );
16127 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16128 assert!(runner.state.judge_skipped);
16129 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16130 assert!(
16131 crate::run::read_artifact(&runner.state, "cand-A.patch")
16132 .unwrap()
16133 .contains("owned edits")
16134 );
16135 assert!(wt.exists());
16136 assert_eq!(
16137 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16138 .await
16139 .unwrap(),
16140 1
16141 );
16142 runner.state = RunState::load(&runner.state.id).unwrap();
16143 }
16144 }
16145
16146 #[tokio::test]
16147 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16148 let (_tmp, mut runner) = preservation_resume_fixture().await;
16149 let wt = runner.state.candidates[0].worktree.clone();
16150 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16151 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16152 let index_lock = PathBuf::from(
16153 git::git(
16154 &wt,
16155 &[
16156 "rev-parse",
16157 "--path-format=absolute",
16158 "--git-path",
16159 "index.lock",
16160 ],
16161 )
16162 .await
16163 .unwrap(),
16164 );
16165 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16166 runner.state.schema = 14;
16167 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16168 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16169 assert!(runner.implement().await.is_err());
16170 assert!(runner.after_implement().is_err());
16171 assert!(runner.fold_losers().await.is_err());
16172 assert_eq!(
16173 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16174 "failed seat edits"
16175 );
16176 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16177 runner.state = RunState::load(&runner.state.id).unwrap();
16178 std::fs::remove_file(index_lock).unwrap();
16179 assert!(runner.implement().await.is_err());
16181 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16182 assert!(runner.state.candidates[0].empty);
16183 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16184 assert!(
16185 crate::run::read_artifact(&runner.state, "cand-A.patch")
16186 .unwrap()
16187 .contains("failed seat edits")
16188 );
16189 assert!(wt.exists());
16190 }
16191
16192 #[tokio::test]
16193 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16194 let (_tmp, mut runner) = preservation_resume_fixture().await;
16195 let repo = runner.state.repo.clone();
16196 let common = git::git(
16197 &repo,
16198 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16199 )
16200 .await
16201 .unwrap();
16202 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16203 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16204 assert!(runner.fold_losers().await.is_err());
16205 assert!(!runner.state.candidates[0].worktree.exists());
16206 assert!(!runner.state.candidates[0].folded);
16207 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16208 runner.state = RunState::load(&runner.state.id).unwrap();
16209 std::fs::remove_file(branch_lock).unwrap();
16210 let judge = runner.state.worktree_root().join("judge-1");
16211 git::worktree_add_detached(&repo, &judge, "main")
16212 .await
16213 .unwrap();
16214 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16215 runner.roles.judges = vec![spec("judge")];
16216 assert!(runner.fold_losers().await.is_err());
16217 runner.state = RunState::load(&runner.state.id).unwrap();
16218 assert!(
16219 runner.state.candidates[0].folded,
16220 "candidate progress saved before judge failure"
16221 );
16222 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16223 assert!(judge.join("uncommitted").exists());
16224 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16225 runner.fold_losers().await.unwrap();
16226 assert!(!judge.exists());
16227 runner.fold_losers().await.unwrap();
16228 }
16229
16230 #[tokio::test]
16231 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16232 let (_tmp, mut runner) = preservation_resume_fixture().await;
16233 runner.state.candidates.retain(|c| c.label == 'B');
16234 runner.state.tally = None;
16235 runner.state.judge_skipped = false;
16236 runner.state.prep_complete = Some(true);
16237 runner.state.config.graph.advise = false;
16238 runner.state.config.graph.selected_implementer = Some("alpha".into());
16239 let wt = runner.state.candidates[0].worktree.clone();
16240 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16241 runner
16242 .state
16243 .candidate_preservation
16244 .insert('B', "awaiting persistence".into());
16245 let index_lock = git::git(
16246 &wt,
16247 &[
16248 "rev-parse",
16249 "--path-format=absolute",
16250 "--git-path",
16251 "index.lock",
16252 ],
16253 )
16254 .await
16255 .unwrap();
16256 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16257 runner.state.save().unwrap();
16258 assert!(runner.execute_graph().await.is_err());
16259 assert_eq!(runner.state.status, RunStatus::Blocked);
16260 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16261 std::fs::remove_file(index_lock).unwrap();
16262 for _ in 0..2 {
16265 runner.state = RunState::load(&runner.state.id).unwrap();
16266 runner.pause.park();
16267 runner.execute_graph().await.unwrap();
16268 assert_eq!(runner.state.status, RunStatus::Implementing);
16269 assert!(runner.state.candidate_preservation.is_empty());
16270 assert_eq!(runner.state.candidates[0].commits, 2);
16271 assert!(runner.state.tally.is_none());
16272 }
16273 runner.implement().await.unwrap();
16274 assert_eq!(runner.state.status, RunStatus::Judging);
16275 runner.state.status = RunStatus::Blocked;
16277 runner.state.candidates[0].failed = Some("implementation failed".into());
16278 runner.state.save().unwrap();
16279 let error = runner.execute_graph().await.unwrap_err();
16280 assert!(
16281 error
16282 .to_string()
16283 .contains("selected implementation is held")
16284 );
16285 }
16286
16287 #[tokio::test]
16288 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16289 let (_tmp, mut runner) = preservation_resume_fixture().await;
16290 let repo = runner.state.repo.clone();
16291 let candidate = runner.state.candidates[0].clone();
16292 let registered = git::worktree_holding(&repo, &candidate.branch)
16295 .await
16296 .unwrap()
16297 .unwrap();
16298 assert_eq!(
16299 registered.canonicalize().unwrap(),
16300 candidate.worktree.canonicalize().unwrap()
16301 );
16302 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16303 assert_eq!(
16304 git::worktree_holding(&repo, &candidate.branch)
16305 .await
16306 .unwrap(),
16307 Some(registered)
16308 );
16309 assert!(
16310 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16311 "the stale registration initially keeps the branch checked out"
16312 );
16313 runner.fold_losers().await.unwrap();
16314 assert!(runner.state.candidates[0].folded);
16315 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16316 assert_eq!(
16317 git::worktree_holding(&repo, &candidate.branch)
16318 .await
16319 .unwrap(),
16320 None
16321 );
16322 runner.state = RunState::load(&runner.state.id).unwrap();
16323 runner.fold_losers().await.unwrap();
16324 }
16325
16326 #[tokio::test]
16327 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16328 let (_tmp, mut runner) = preservation_resume_fixture().await;
16329 let repo = runner.state.repo.clone();
16330 let candidate = runner.state.candidates[0].clone();
16331 let winner = runner.state.candidates[1].clone();
16332 let registered = git::worktree_holding(&repo, &candidate.branch)
16333 .await
16334 .unwrap()
16335 .unwrap();
16336 assert_eq!(
16337 registered.canonicalize().unwrap(),
16338 candidate.worktree.canonicalize().unwrap()
16339 );
16340 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16341 assert_eq!(
16342 git::worktree_holding(&repo, &candidate.branch)
16343 .await
16344 .unwrap(),
16345 Some(registered)
16346 );
16347 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16348
16349 for _ in 0..2 {
16350 fold_run(&mut runner.state, false, &crate::run::home())
16351 .await
16352 .unwrap();
16353 runner.state = RunState::load(&runner.state.id).unwrap();
16354 assert!(runner.state.candidates[0].folded);
16355 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16356 assert_eq!(
16357 git::worktree_holding(&repo, &candidate.branch)
16358 .await
16359 .unwrap(),
16360 None
16361 );
16362 assert!(!runner.state.candidates[1].folded);
16363 assert!(winner.worktree.exists());
16364 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16365 }
16366 }
16367
16368 #[tokio::test]
16369 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16370 ask_test_home();
16371 let tmp = tempfile::tempdir().unwrap();
16372 let repo = tmp.path().join("repo");
16373 std::fs::create_dir(&repo).unwrap();
16374 init_repo(&repo);
16375 let mut runner = runner_at(RunStatus::Implementing);
16376 runner.state.repo = repo.clone();
16377 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16378 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16379 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16380 let branch = format!("candidate-{label}");
16381 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16382 git::worktree_add_branch(&repo, &wt, &branch, "main")
16383 .await
16384 .unwrap();
16385 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16386 git::git(&wt, &["add", "README.md"]).await.unwrap();
16387 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16388 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16389 runner.state.candidates.push(Candidate {
16390 index,
16391 label,
16392 agent: "alpha".to_owned(),
16393 branch,
16394 worktree: wt,
16395 summary: "completed implementation".to_owned(),
16396 stat: String::new(),
16397 files: 0,
16398 commits: 0,
16399 empty: false,
16400 failed: None,
16401 verified_noop: None,
16402 duration_ms: 42,
16403 folded: false,
16404 });
16405 runner
16406 .state
16407 .candidate_preservation
16408 .insert(label, "awaiting persistence".to_owned());
16409 }
16410 let wt = runner.state.candidates[0].worktree.clone();
16411 let index_lock = git::git(
16412 &wt,
16413 &[
16414 "rev-parse",
16415 "--path-format=absolute",
16416 "--git-path",
16417 "index.lock",
16418 ],
16419 )
16420 .await
16421 .unwrap();
16422 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16423 runner.state.save().unwrap();
16424 assert!(runner.persist_candidates().await.is_err());
16425 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16426 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16427 assert_eq!(
16428 runner.state.candidates[1].commits, 1,
16429 "other results are still collected"
16430 );
16431 assert!(
16432 !runner.state.candidates[0].empty,
16433 "failed rescue is not no change"
16434 );
16435 assert_eq!(
16436 runner.state.candidates[0].summary,
16437 "completed implementation"
16438 );
16439 assert!(runner.after_implement().is_err());
16440 let home = tmp.path().join("fold-home");
16441 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16442 assert!(wt.join("new-file").exists());
16443 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16444 assert!(recovery.join("staged.patch").exists());
16445 assert!(recovery.join("unstaged.patch").exists());
16446 assert_eq!(
16447 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16448 "untracked edits\n"
16449 );
16450 runner.state.begin_pass();
16451 runner.state.end_pass_parked(ParkCause::Upgrade {
16452 from: "1.0".into(),
16453 to: Some("1.1".into()),
16454 });
16455 runner.state.save().unwrap();
16456 runner.state = RunState::load(&runner.state.id).unwrap();
16457 assert!(matches!(
16458 runner.state.passes.last().unwrap().end,
16459 Some(crate::run::PassEnd::Parked {
16460 cause: ParkCause::Upgrade { .. }
16461 })
16462 ));
16463 std::fs::remove_file(index_lock).unwrap();
16464 runner.implement().await.unwrap();
16466 assert!(runner.state.candidate_preservation.is_empty());
16467 assert!(
16468 runner
16469 .state
16470 .candidates
16471 .iter()
16472 .all(|c| c.commits == 1 && !c.empty)
16473 );
16474 assert!(git::is_clean(&wt).await.unwrap());
16475 assert!(
16476 crate::run::read_artifact(&runner.state, "cand-A.patch")
16477 .unwrap()
16478 .contains("later tracked edits")
16479 );
16480 }
16481
16482 #[tokio::test]
16483 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16484 ask_test_home();
16485 let tmp = tempfile::tempdir().unwrap();
16486 let repo = tmp.path().join("repo");
16487 std::fs::create_dir(&repo).unwrap();
16488 init_repo(&repo);
16489 let wt = tmp.path().join("candidate");
16490 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16491 .await
16492 .unwrap();
16493 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16494 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16495 assert!(wt.join("uncommitted").exists());
16496 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16497 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16498 let mut runner = runner_at(RunStatus::Implementing);
16499 runner.state.repo = repo.clone();
16500 runner.state.base_commit = "missing-base".to_owned();
16501 runner.state.candidates.push(Candidate {
16502 index: 0,
16503 label: 'A',
16504 agent: "alpha".into(),
16505 branch: "candidate".into(),
16506 worktree: wt.clone(),
16507 summary: "completed".into(),
16508 stat: String::new(),
16509 files: 0,
16510 commits: 0,
16511 empty: false,
16512 failed: None,
16513 verified_noop: None,
16514 duration_ms: 1,
16515 folded: false,
16516 });
16517 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16518 assert!(
16519 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16520 .await
16521 .is_err()
16522 );
16523 assert!(!runner.state.candidates[0].folded);
16524 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16525 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16526 runner
16527 .state
16528 .candidate_preservation
16529 .insert('A', "pending".into());
16530 assert!(runner.persist_candidates().await.is_err());
16531 assert!(!runner.state.candidates[0].empty);
16532 assert!(!runner.state.candidates[0].folded);
16533 assert!(wt.exists());
16534 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16535 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16536 std::fs::create_dir_all(&patch_path).unwrap();
16537 assert!(
16538 runner.persist_candidates().await.is_err(),
16539 "artifact failure stays pending"
16540 );
16541 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16542 assert!(!runner.state.candidates[0].empty);
16543 std::fs::remove_dir(patch_path).unwrap();
16544 runner.persist_candidates().await.unwrap();
16545 assert!(
16546 runner.state.candidates[0].empty,
16547 "only successful inspection confirms no change"
16548 );
16549 }
16550
16551 fn init_repo(dir: &Path) {
16552 let run = |args: &[&str]| {
16553 let out = std::process::Command::new("git")
16554 .args(args)
16555 .current_dir(dir)
16556 .quiet()
16557 .output()
16558 .expect("spawn git");
16559 assert!(
16560 out.status.success(),
16561 "git {args:?} failed: {}",
16562 String::from_utf8_lossy(&out.stderr)
16563 );
16564 };
16565 run(&["init", "-b", "main"]);
16566 run(&["config", "user.name", "magi test"]);
16567 run(&["config", "user.email", "magi@example.com"]);
16568 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16569 run(&["add", "-A"]);
16570 run(&["commit", "-m", "init"]);
16571 }
16572
16573 fn ask_test_home() {
16581 crate::run::pin_test_home();
16582 }
16583
16584 #[test]
16585 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16586 ask_test_home();
16587 let mut runner = runner_at(RunStatus::Prep);
16588 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16589 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16592 let error = guard
16593 .finish(
16594 &mut runner.state,
16595 Err(
16596 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16597 .into(),
16598 ),
16599 )
16600 .unwrap_err();
16601 assert_eq!(
16602 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16603 std::io::ErrorKind::PermissionDenied
16604 );
16605 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16606 let message = format!("{error:#}");
16607 assert!(message.contains("original refusal"));
16608 assert!(!message.contains("could not save terminal"));
16609 assert_eq!(
16610 error.chain().count(),
16611 2,
16612 "only startup attribution wraps the original error"
16613 );
16614 let saved = RunState::load(&runner.state.id).unwrap();
16615 assert_eq!(saved.status, RunStatus::Blocked);
16616 assert!(!saved.startup.unwrap().complete);
16617 }
16618
16619 #[test]
16620 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16621 ask_test_home();
16622 let mut runner = runner_at(RunStatus::Prep);
16623 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16624 let id = runner.state.id.clone();
16625 drop(guard);
16626 let saved = RunState::load(&id).unwrap();
16627 assert_eq!(saved.status, RunStatus::Blocked);
16628 assert!(!saved.startup.as_ref().unwrap().complete);
16629 assert!(
16630 saved
16631 .startup
16632 .unwrap()
16633 .failure
16634 .unwrap()
16635 .contains("interrupted")
16636 );
16637 assert!(Runner::resume(&id).is_err());
16638 }
16639
16640 #[test]
16641 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16642 ask_test_home();
16643 let mut runner = runner_at(RunStatus::Prep);
16644 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16645 let error = guard
16646 .finish(
16647 &mut runner.state,
16648 Err(
16649 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16650 .into(),
16651 ),
16652 )
16653 .unwrap_err();
16654 assert_eq!(
16655 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16656 std::io::ErrorKind::PermissionDenied
16657 );
16658 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16659 let mut task = crate::queue::Task::new(
16660 "review".into(),
16661 "criteria".into(),
16662 runner.state.repo.clone(),
16663 crate::queue::Source::Human,
16664 );
16665 task.record_startup_failure(failure.clone());
16666 task.record_startup_failure(failure.clone());
16667 assert_eq!(task.startup_failures.len(), 1);
16668 assert!(task.runs.is_empty());
16669 assert_eq!(task.attempts, 0);
16670 }
16671
16672 #[cfg(unix)]
16673 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16674 ask_test_home();
16675 let tmp = tempfile::tempdir().unwrap();
16676 let repo = tmp.path().join("repo");
16677 std::fs::create_dir(&repo).unwrap();
16678 init_repo(&repo);
16679 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16680 let origin = tmp.path().join("origin.git");
16681 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16682 .await
16683 .unwrap();
16684 git::git(
16685 &repo,
16686 &["remote", "add", "origin", origin.to_str().unwrap()],
16687 )
16688 .await
16689 .unwrap();
16690 git::git(&repo, &["checkout", "-b", "fix/retained"])
16691 .await
16692 .unwrap();
16693 std::fs::write(repo.join("retained"), "retained change").unwrap();
16694 git::git(&repo, &["add", "."]).await.unwrap();
16695 git::git(&repo, &["commit", "-m", "retained implementation"])
16696 .await
16697 .unwrap();
16698 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16699 git::git(&repo, &["checkout", "main"]).await.unwrap();
16700 std::fs::write(repo.join("base-change"), "new base").unwrap();
16701 git::git(&repo, &["add", "."]).await.unwrap();
16702 git::git(&repo, &["commit", "-m", "base moved"])
16703 .await
16704 .unwrap();
16705 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16706 git::git(&repo, &["checkout", "fix/retained"])
16707 .await
16708 .unwrap();
16709 let mut runner = runner_at(RunStatus::Voting);
16710 runner.state.repo = repo.clone();
16711 runner.state.base_commit = base;
16712 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16713 runner.state.config.disk.min_free_bytes = 0;
16714 runner.state.config.graph.review_rounds = 1;
16715 runner.state.config.verify.e2e = vec!["true".into()];
16716 runner.state.config.verify.gate = vec!["true".into()];
16717 runner.state.candidates = vec![Candidate {
16718 index: 0,
16719 label: 'A',
16720 agent: "fixture".into(),
16721 branch: "fix/retained".into(),
16722 worktree: repo.clone(),
16723 summary: String::new(),
16724 stat: String::new(),
16725 files: 1,
16726 commits: 1,
16727 empty: false,
16728 failed: None,
16729 verified_noop: None,
16730 duration_ms: 0,
16731 folded: false,
16732 }];
16733 runner.state.tally = Some(Tally {
16734 first_choice: BTreeMap::new(),
16735 borda: BTreeMap::new(),
16736 winner: 'A',
16737 rankings: 0,
16738 unanimous_initial: true,
16739 deliberated: false,
16740 changed_votes: 0,
16741 unanimous_final: true,
16742 tie_break: None,
16743 judges: 0,
16744 present: 0,
16745 quorum: 0,
16746 met_quorum: true,
16747 uncontested: Some("review-only".into()),
16748 });
16749 let mut task = crate::queue::Task::new(
16750 "fixture".into(),
16751 "fixture".into(),
16752 repo,
16753 crate::queue::Source::Human,
16754 );
16755 task.link_run(&runner.state.id);
16756 crate::queue::Queue::open().put(&mut task).unwrap();
16757 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16758 runner.state.base_sync = Some(BaseSync {
16759 tip: String::new(),
16760 behind: 1,
16761 attempts: BASE_SYNC_ROUNDS,
16762 conflict: None,
16763 already_in: None,
16764 });
16765 runner.sync_to_base().await.unwrap();
16766 assert_eq!(runner.state.status, RunStatus::Blocked);
16767 (tmp, runner, head, task.id)
16768 }
16769
16770 #[cfg(unix)]
16771 #[tokio::test]
16772 async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16773 for routed in [false, true] {
16774 let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16775 let home = crate::run::home();
16776 let queue = crate::queue::Queue::open();
16777 let task = queue.get(&owner).unwrap();
16778 runner.state.task_instruction = Some(task.instruction.clone());
16779 runner.state.driver_exited = true;
16780 runner.state.save().unwrap();
16781 let questions = crate::ask::Questions::at(home.join("questions"));
16782 let mut q = crate::ask::Question::new(
16783 runner.state.id.clone(),
16784 "base-recovery".into(),
16785 "owner".into(),
16786 "Recover bounded base movement?".into(),
16787 String::new(),
16788 vec!["recover".into()],
16789 );
16790 q.actions.insert(
16791 "recover".into(),
16792 crate::ask::ChoiceAction::RecoverBase {
16793 run: runner.state.id.clone(),
16794 stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16795 episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16796 },
16797 );
16798 q.routed_task = Some(owner.clone());
16799 q.answer(crate::ask::Answer::Choice("recover".into()))
16800 .unwrap();
16801 questions.put(&mut q).unwrap();
16802 crate::base_recovery_control::recover(
16803 &queue,
16804 &home,
16805 crate::base_recovery_control::Request {
16806 run: runner.state.id.clone(),
16807 head: head.clone(),
16808 actor: "decisions".into(),
16809 request: q.id.clone(),
16810 decision: Some((owner.clone(), q.answered_at.unwrap())),
16811 expected_stop: Some((
16812 runner.state.base_sync_recovery.stop.clone().unwrap(),
16813 runner.state.base_sync_recovery.episodes.last().unwrap().id,
16814 )),
16815 },
16816 )
16817 .await
16818 .unwrap();
16819 if routed {
16820 questions
16821 .update(&q.id, |q| {
16822 q.routed_task = Some("another-owner".into());
16823 Ok(())
16824 })
16825 .unwrap();
16826 } else {
16827 let mut task = queue.get(&owner).unwrap();
16828 task.hold_manual(Some("new owner decision".into()));
16829 task.status = crate::queue::TaskStatus::Queued;
16831 queue.put(&mut task).unwrap();
16832 }
16833 runner.state = RunState::load(&runner.state.id).unwrap();
16834 runner.sync_to_base().await.unwrap();
16835 let approval = runner
16836 .state
16837 .base_sync_recovery
16838 .authorizations
16839 .last()
16840 .unwrap();
16841 assert!(approval.consumed);
16842 assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16843 "routing"
16844 } else {
16845 "owner decision"
16846 }));
16847 assert_eq!(
16848 runner.state.base_sync.as_ref().unwrap().attempts,
16849 BASE_SYNC_ROUNDS
16850 );
16851 assert_eq!(
16852 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16853 head
16854 );
16855 }
16856 }
16857
16858 #[cfg(unix)]
16859 #[tokio::test]
16860 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16861 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16862 if std::env::var_os(CHILD).is_none() {
16863 let home = tempfile::tempdir().unwrap();
16864 let output = std::process::Command::new(std::env::current_exe().unwrap())
16865 .args([
16866 "--exact",
16867 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16868 "--nocapture",
16869 ])
16870 .env(CHILD, "1")
16871 .env("MAGI_HOME", home.path())
16872 .env("MAGI_WORKER_HOME", home.path())
16873 .env("MAGI_WORKER_ID", "abcdef")
16874 .output()
16875 .unwrap();
16876 assert!(
16877 output.status.success()
16878 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16879 "{}\n{}",
16880 String::from_utf8_lossy(&output.stdout),
16881 String::from_utf8_lossy(&output.stderr)
16882 );
16883 return;
16884 }
16885 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16886 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16887 let home = crate::run::home();
16888 let queue = crate::queue::Queue::open();
16889 let mut owner = queue.get(&task).unwrap();
16890 owner.release_operator("test-release", None);
16891 owner.start(runner.state.id.clone());
16892 queue.put(&mut owner).unwrap();
16893 runner.state.driver_exited = true;
16894 runner.state.task_instruction = Some(owner.instruction.clone());
16895 runner.state.save().unwrap();
16896 let _claim = queue.claim(&task).unwrap();
16897 let reserve = |id: &str| {
16898 let directory = home.join("workers/jobs").join(id);
16899 std::fs::create_dir_all(&directory).unwrap();
16900 let job = crate::worker::Job {
16901 id: id.into(),
16902 sequence: 1,
16903 kind: "task".into(),
16904 key: format!("task:{task}"),
16905 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16906 generation: "fixture-generation".into(),
16907 version: None,
16908 status: crate::worker::Status::Running,
16909 result: None,
16910 pid: Some(std::process::id()),
16911 created_at: 0,
16912 finished_at: None,
16913 };
16914 std::fs::write(
16915 directory.join("job.json"),
16916 serde_json::to_vec(&job).unwrap(),
16917 )
16918 .unwrap();
16919 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16920 };
16921 let _own = reserve("abcdef");
16922 let other = reserve("fedcba");
16923 assert!(
16924 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16925 .await
16926 .is_err()
16927 );
16928 assert!(
16929 RunState::load(&runner.state.id)
16930 .unwrap()
16931 .base_sync_recovery
16932 .authorizations
16933 .is_empty()
16934 );
16935 drop(other);
16936 assert!(crate::daemon::is_working_on(
16937 &home,
16938 &runner.state.id,
16939 Timestamp::now()
16940 ));
16941 assert!(
16942 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16943 );
16944 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16945 .await
16946 .unwrap();
16947 assert!(
16948 RunState::load(&runner.state.id)
16949 .unwrap()
16950 .base_recovery_pending()
16951 );
16952 }
16953
16954 #[cfg(unix)]
16955 #[tokio::test]
16956 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16957 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16958 let home = crate::run::home();
16959 runner.state.driver_pid = Some(std::process::id());
16960 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16961 runner.state.driver_exited = false;
16962 runner.state.begin_pass();
16963 runner.state.save().unwrap();
16964 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16965 driver.record(&runner.state).unwrap();
16966 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16967 drop(driver);
16969 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16970 assert_eq!(
16971 runner.state.liveness_under(&home, false),
16972 crate::run::Liveness::Dead
16973 );
16974 let queue = crate::queue::Queue::open();
16975 let _claim = queue.claim(&task).unwrap();
16976 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16977 .await
16978 .unwrap();
16979 let disk = RunState::load(&runner.state.id).unwrap();
16980 assert!(!disk.driver_exited);
16981 assert!(disk.base_recovery_pending());
16982 }
16983
16984 #[cfg(unix)]
16985 #[tokio::test]
16986 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16987 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16988 if std::env::var_os(CHILD).is_none() {
16989 use std::os::unix::fs::PermissionsExt;
16990 let tmp = tempfile::tempdir().unwrap();
16991 let gh = tmp.path().join("gh");
16992 std::fs::write(&gh, r#"#!/bin/sh
16993case "$1 $2" in
16994 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16995 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16996 *) exit 2 ;;
16997esac
16998"#).unwrap();
16999 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17000 let mut paths = vec![tmp.path().to_path_buf()];
17001 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17002 let output = std::process::Command::new(std::env::current_exe().unwrap())
17003 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17004 .env(CHILD, "1")
17005 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17006 .env("PATH", std::env::join_paths(paths).unwrap())
17007 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17008 .output().unwrap();
17009 assert!(
17010 output.status.success()
17011 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17012 "{}\n{}",
17013 String::from_utf8_lossy(&output.stdout),
17014 String::from_utf8_lossy(&output.stderr)
17015 );
17016 return;
17017 }
17018 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17019 let repo = runner.state.repo.clone();
17020 git::git(&repo, &["checkout", "main"]).await.unwrap();
17021 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17022 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17023 git::git(&repo, &["checkout", "fix/retained"])
17024 .await
17025 .unwrap();
17026 runner.state.pr = Some(crate::run::PrRecord {
17027 url: "https://example.test/project/pull/1".into(),
17028 number: 1,
17029 state: "open".into(),
17030 checks: "unknown".into(),
17031 round: 0,
17032 rounds: 1,
17033 red_at_merge: Vec::new(),
17034 });
17035 runner.sync_to_base().await.unwrap();
17036 assert_eq!(runner.state.status, RunStatus::Blocked);
17037 assert_eq!(
17038 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17039 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17040 );
17041 assert!(runner.state.base_sync_closure_pending());
17042 assert!(crate::handover::can_resume(&runner.state));
17043 let queue = crate::queue::Queue::open();
17044 assert!(!crate::base_sync_recovery::held_task(
17045 &queue,
17046 &queue.get(&task).unwrap()
17047 ));
17048 assert!(runner.state.authorize_base_sync(&task).is_err());
17049 runner.state.base_sync_recovery.stop = None;
17051 runner.state.status = RunStatus::Voting;
17052 runner.state.gate_ran = true;
17053 runner.state.config.graph.review_rounds = 0;
17054 runner.state.save().unwrap();
17055 assert!(crate::handover::can_resume(&runner.state));
17056 runner.state = RunState::load(&runner.state.id).unwrap();
17057 runner.execute_graph().await.unwrap();
17058 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17059 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17060 assert_eq!(
17061 runner.state.base_sync.as_ref().unwrap().attempts,
17062 BASE_SYNC_ROUNDS
17063 );
17064 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17065 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17066 assert!(runner.state.reviews.is_empty());
17067 assert!(runner.state.gate.is_empty());
17068 }
17069
17070 #[cfg(unix)]
17071 #[tokio::test]
17072 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17073 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17074 runner.state.driver_exited = true;
17075 runner.state.save().unwrap();
17076 let home = crate::run::home();
17077 let queue = crate::queue::Queue::open();
17078 let _claim = queue.claim(&task).unwrap();
17079 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17080 assert!(
17081 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17082 .await
17083 .is_err()
17084 );
17085 drop(lease);
17086 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17087 assert!(
17088 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17089 .await
17090 .is_err()
17091 );
17092 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17093 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17094 .await
17095 .unwrap();
17096 let disk = RunState::load(&runner.state.id).unwrap();
17097 assert!(disk.base_recovery_pending());
17098 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17099 assert!(
17100 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17101 .await
17102 .is_err()
17103 );
17104 assert_eq!(
17105 RunState::load(&runner.state.id)
17106 .unwrap()
17107 .base_sync_recovery
17108 .authorizations
17109 .len(),
17110 1
17111 );
17112 let mut owner = queue.get(&task).unwrap();
17113 owner.runs.push("20260101-000000-ffff".into());
17114 queue.put(&mut owner).unwrap();
17115 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17116 .await
17117 .unwrap_err();
17118 assert!(error.to_string().contains("run task ownership changed"));
17119 }
17120
17121 #[cfg(unix)]
17122 #[tokio::test]
17123 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17124 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17125 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17126 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17127 legacy["status"] = serde_json::json!("voting");
17128 runner.state = serde_json::from_value(legacy).unwrap();
17129 runner.state.save().unwrap();
17130 for _ in 0..3 {
17131 runner.execute_graph().await.unwrap();
17132 assert_eq!(runner.state.status, RunStatus::Blocked);
17133 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17134 assert!(!crate::handover::can_resume(&runner.state));
17135 assert_eq!(
17136 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17137 head
17138 );
17139 assert!(runner.state.reviews.is_empty());
17140 assert!(!runner.state.gate_ran);
17141 }
17142 let notice = crate::notices::run_ended(&runner.state).unwrap();
17143 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17144 runner.state.begin_pass();
17145 runner.state.status = RunStatus::Voting;
17146 runner.deliberate().await.unwrap();
17147 runner.vote().await.unwrap();
17148 runner.finish_after_tally().await.unwrap();
17149 assert_eq!(runner.state.status, RunStatus::Blocked);
17150 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17151 assert_eq!(
17152 crate::notices::run_ended(&runner.state).unwrap().since,
17153 notice.since
17154 );
17155 }
17156
17157 #[cfg(unix)]
17158 #[tokio::test]
17159 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17160 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17161 for detail in [
17162 "fix/retained conflicts with origin/main and did not rebase: conflict",
17163 "rebased fix/retained locally but origin refused the push",
17164 "unknown failure",
17165 ] {
17166 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17167 runner.state.record_base_sync_stop(&head);
17168 assert!(runner.state.authorize_base_sync(&task).is_err());
17169 runner.state.status = RunStatus::Voting;
17170 runner.execute_graph().await.unwrap();
17171 assert_eq!(runner.state.status, RunStatus::Blocked);
17172 assert!(!crate::handover::can_resume(&runner.state));
17173 assert_eq!(
17174 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17175 head
17176 );
17177 }
17178 }
17179
17180 #[cfg(unix)]
17181 #[tokio::test]
17182 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17183 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17184 runner.state.authorize_base_sync(&task).unwrap();
17185 assert!(runner.state.consume_base_recovery(&head));
17186 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17187 runner.state.reserve_base_operation(&head, &tip);
17188 runner.state.save().unwrap();
17189 runner.state = RunState::load(&runner.state.id).unwrap();
17190 assert!(!runner.state.consume_base_recovery(&head));
17191 runner.sync_to_base().await.unwrap();
17192 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17193 assert_eq!(runner.state.base_episode_used(), 1);
17194 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17195 assert!(
17196 runner
17197 .state
17198 .base_sync_recovery
17199 .operations
17200 .last()
17201 .unwrap()
17202 .finished
17203 );
17204 assert!(!runner.state.base_sync_blocked());
17205 runner.sync_to_base().await.unwrap();
17206 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17207 }
17208
17209 #[cfg(unix)]
17210 #[tokio::test]
17211 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17212 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17213 let original = runner
17215 .state
17216 .base_sync
17217 .as_ref()
17218 .unwrap()
17219 .conflict
17220 .clone()
17221 .unwrap();
17222 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17223 original
17224 .replace("synchronization(s)", "rebase(s)")
17225 .replace("synchronizing", "rebasing"),
17226 );
17227 runner.state.base_sync_recovery = Default::default();
17228 runner.state.record_base_sync_stop(&head);
17229 assert_eq!(
17230 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17231 crate::base_sync_recovery::StopKind::Exhausted
17232 );
17233 runner.state.authorize_base_sync(&task).unwrap();
17234 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17235 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17236 .as_object_mut()
17237 .unwrap();
17238 auth.remove("interval");
17239 auth.remove("scope");
17240 runner.state = serde_json::from_value(legacy).unwrap();
17241 assert!(runner.state.consume_base_recovery(&head));
17242 assert_eq!(runner.state.base_episode_used(), 3);
17243 runner.sync_to_base().await.unwrap();
17244 assert_eq!(runner.state.base_episode_used(), 4);
17245 runner.state.resource_block = Some(crate::run::ResourceBlock {
17246 node: "gate".into(),
17247 kind: "cache".into(),
17248 detail: "legacy wait".into(),
17249 at: Timestamp::now(),
17250 cause: crate::run::ResourceBlockCause::Unrecorded,
17251 });
17252 runner.state.save().unwrap();
17253 assert!(
17254 runner
17255 .state
17256 .base_sync_recovery
17257 .episodes
17258 .last()
17259 .unwrap()
17260 .boundary
17261 .is_none()
17262 );
17263 runner.state.resource_block = None;
17264 advance_base_fixture(&runner).await;
17265 runner.sync_to_base().await.unwrap();
17266 assert!(runner.state.base_sync_blocked());
17267 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17268 }
17269
17270 #[cfg(unix)]
17271 async fn advance_base_fixture(runner: &Runner) {
17272 let repo = &runner.state.repo;
17273 for args in [
17274 vec!["checkout", "main"],
17275 vec!["commit", "--allow-empty", "-m", "base advanced"],
17276 vec!["push", "origin", "main"],
17277 vec!["checkout", "fix/retained"],
17278 ] {
17279 git::git(repo, &args).await.unwrap();
17280 }
17281 }
17282
17283 #[cfg(unix)]
17284 fn fresh_base_interval_fixture(runner: &mut Runner) {
17285 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17286 runner.state.base_sync.as_mut().unwrap().conflict = None;
17287 runner.state.base_sync_recovery = Default::default();
17288 runner.state.status = RunStatus::Voting;
17289 }
17290
17291 #[cfg(unix)]
17292 #[tokio::test]
17293 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17294 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17295 fresh_base_interval_fixture(&mut runner);
17296 runner.state.begin_pass();
17297 runner.sync_to_base().await.unwrap();
17298 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17299 runner.state.resource_block = Some(crate::run::ResourceBlock {
17300 node: "gate".into(),
17301 kind: "build cache".into(),
17302 detail: "busy".into(),
17303 at: Timestamp::now(),
17304 cause: crate::run::ResourceBlockCause::Timeout {
17305 waited_secs: 1,
17306 limit_secs: 1,
17307 },
17308 });
17309 runner.state.save().unwrap();
17310 let queue = crate::queue::Queue::open();
17311 let mut owner = queue.get(&task).unwrap();
17312 runner.state.task_instruction = Some(owner.instruction.clone());
17313 owner.release_operator("fixture-owner", None);
17314 queue.put(&mut owner).unwrap();
17315 let approval = owner.resumption.clone().unwrap();
17316 runner.state.resource_block = None;
17317 runner.state.save().unwrap();
17318 runner.state = RunState::load(&runner.state.id).unwrap();
17319 advance_base_fixture(&runner).await;
17320 runner.sync_to_base().await.unwrap();
17321 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17322 assert_eq!(runner.state.base_episode_used(), 1);
17323 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17324 assert!(
17325 runner.state.base_sync_recovery.episodes[0]
17326 .boundary
17327 .as_ref()
17328 .unwrap()
17329 .consumed
17330 );
17331 assert!(
17332 !runner
17333 .state
17334 .renew_base_episode(&task, &approval, &head, "base")
17335 );
17336 for _ in 0..3 {
17337 owner.release_operator("fixture-owner", None);
17338 queue.put(&mut owner).unwrap();
17339 runner.state = RunState::load(&runner.state.id).unwrap();
17340 runner.state.begin_pass();
17341 runner.sync_to_base().await.unwrap();
17342 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17343 assert_eq!(runner.state.base_episode_used(), 1);
17344 }
17345 runner.state.resource_block = Some(crate::run::ResourceBlock {
17346 node: "gate".into(),
17347 kind: "disk pressure".into(),
17348 detail: "free space required".into(),
17349 at: Timestamp::now(),
17350 cause: crate::run::ResourceBlockCause::Other,
17351 });
17352 runner.state.end_pass_parked(ParkCause::Interrupt {
17353 reason: "disk pressure".into(),
17354 });
17355 runner.state.save().unwrap();
17356 owner.release_operator("fixture-owner-after-disk", None);
17357 queue.put(&mut owner).unwrap();
17358 runner.state.resource_block = None;
17359 advance_base_fixture(&runner).await;
17360 runner.sync_to_base().await.unwrap();
17361 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17362 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17363 assert_eq!(runner.state.base_episode_used(), 1);
17364 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17365 }
17366
17367 #[cfg(unix)]
17368 #[tokio::test]
17369 async fn base_recovery_continuous_advancement_stops_at_four() {
17370 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17371 fresh_base_interval_fixture(&mut runner);
17372 for spent in 1..=BASE_SYNC_ROUNDS {
17373 runner.sync_to_base().await.unwrap();
17374 assert!(!runner.state.base_sync_blocked());
17375 assert_eq!(runner.state.base_episode_used(), spent);
17376 advance_base_fixture(&runner).await;
17377 }
17378 runner.sync_to_base().await.unwrap();
17379 assert_eq!(
17380 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17381 crate::base_sync_recovery::StopKind::Exhausted
17382 );
17383 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17384 assert!(stop.unverified_sync);
17385 let action = runner.state.base_sync_next_action();
17386 assert!(action.contains("never reached its gate"));
17387 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17388 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17389 runner.state = RunState::load(&runner.state.id).unwrap();
17390 runner.sync_to_base().await.unwrap();
17391 assert_eq!(runner.state.base_episode_used(), 4);
17392 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17393 }
17394
17395 #[cfg(unix)]
17396 #[tokio::test]
17397 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17398 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17399 runner.state.ensure_base_episode();
17400 runner.state.record_base_progress(&head);
17402 runner.state.base_sync_recovery.stop = None;
17403 runner.state.record_base_sync_stop(&head);
17404 assert!(
17405 !runner
17406 .state
17407 .base_sync_recovery
17408 .stop
17409 .as_ref()
17410 .unwrap()
17411 .unverified_sync
17412 );
17413
17414 runner.state.finish_base_operation(Some(&head));
17415 runner.state.gate_ran = true;
17417 runner.state.gate_head = Some("older-head".into());
17418 runner.state.base_sync_recovery.stop = None;
17419 runner.state.record_base_sync_stop(&head);
17420 runner.state.save().unwrap();
17421 runner.state = RunState::load(&runner.state.id).unwrap();
17422 assert!(
17423 runner
17424 .state
17425 .base_sync_recovery
17426 .stop
17427 .as_ref()
17428 .unwrap()
17429 .unverified_sync
17430 );
17431
17432 runner.state.gate_head = Some(head.clone());
17433 runner.state.base_sync_recovery.stop = None;
17434 runner.state.record_base_sync_stop(&head);
17435 assert!(
17436 !runner
17437 .state
17438 .base_sync_recovery
17439 .stop
17440 .as_ref()
17441 .unwrap()
17442 .unverified_sync
17443 );
17444 assert!(
17445 !runner
17446 .state
17447 .base_sync_next_action()
17448 .contains("never reached its gate")
17449 );
17450
17451 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17453 legacy["base_sync_recovery"]["stop"]
17454 .as_object_mut()
17455 .unwrap()
17456 .remove("unverified_sync");
17457 legacy["base_sync_recovery"]["episodes"][0]
17458 .as_object_mut()
17459 .unwrap()
17460 .remove("last_sync_head");
17461 runner.state = serde_json::from_value(legacy).unwrap();
17462 assert!(
17463 !runner
17464 .state
17465 .base_sync_recovery
17466 .stop
17467 .as_ref()
17468 .unwrap()
17469 .unverified_sync
17470 );
17471 assert!(
17472 runner.state.base_sync_recovery.episodes[0]
17473 .last_sync_head
17474 .is_none()
17475 );
17476 }
17477
17478 #[cfg(unix)]
17479 #[tokio::test]
17480 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17481 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17482 runner.state.authorize_base_sync(&task).unwrap();
17483 assert!(
17484 runner
17485 .state
17486 .base_sync_next_action()
17487 .contains("Approval recorded")
17488 );
17489 assert!(
17490 runner
17491 .state
17492 .base_sync_next_action()
17493 .contains("4/4 in episode 0, 4 lifetime")
17494 );
17495 assert!(runner.state.consume_base_recovery(&head));
17496 assert!(
17497 runner
17498 .state
17499 .base_sync_next_action()
17500 .contains("0/4 in episode 1, 4 lifetime")
17501 );
17502 }
17503
17504 #[cfg(unix)]
17505 #[tokio::test]
17506 async fn base_recovery_changed_head_invalidates_without_spending() {
17507 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17508 runner.state.authorize_base_sync(&task).unwrap();
17509 git::git(
17510 &runner.state.repo,
17511 &["commit", "--allow-empty", "-m", "external head change"],
17512 )
17513 .await
17514 .unwrap();
17515 runner.sync_to_base().await.unwrap();
17516 let a = runner
17517 .state
17518 .base_sync_recovery
17519 .authorizations
17520 .last()
17521 .unwrap();
17522 assert!(a.consumed);
17523 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17524 runner.state = RunState::load(&runner.state.id).unwrap();
17525 assert!(
17526 runner
17527 .state
17528 .base_sync_next_action()
17529 .contains("Approval invalidated")
17530 );
17531 assert!(!runner.state.base_recovery_pending());
17532 assert!(runner.state.base_sync_blocked());
17533 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17534 }
17535
17536 #[cfg(unix)]
17537 #[tokio::test]
17538 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17539 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17540 runner.state.driver_exited = true;
17541 runner.state.save().unwrap();
17542 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17543 .await
17544 .unwrap();
17545 runner.state = RunState::load(&runner.state.id).unwrap();
17546 advance_base_fixture(&runner).await;
17547 runner.sync_to_base().await.unwrap();
17548 let a = runner
17549 .state
17550 .base_sync_recovery
17551 .authorizations
17552 .last()
17553 .unwrap();
17554 assert!(
17555 a.consumed
17556 && a.invalidated
17557 .as_deref()
17558 .unwrap()
17559 .contains("fetched base changed")
17560 );
17561 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17562 assert_eq!(
17563 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17564 head
17565 );
17566 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17567 .await
17568 .unwrap();
17569 runner.state = RunState::load(&runner.state.id).unwrap();
17570 runner.sync_to_base().await.unwrap();
17571 assert!(!runner.state.base_sync_blocked());
17572 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17573 }
17574
17575 #[cfg(unix)]
17576 #[tokio::test]
17577 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17578 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17579 runner.state.authorize_base_sync(&task).unwrap();
17580 assert!(runner.state.consume_base_recovery(&head));
17581 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17582 runner.state.reserve_base_operation(&head, &tip);
17583 runner.state.save().unwrap();
17584 let scratch = runner.state.dir().join("base-sync");
17585 assert_eq!(
17586 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17587 .await
17588 .unwrap(),
17589 crate::rebase::Rebased::Applied
17590 );
17591 runner.state = RunState::load(&runner.state.id).unwrap();
17592 runner.sync_to_base().await.unwrap();
17593 assert!(
17594 runner
17595 .state
17596 .base_sync_recovery
17597 .operations
17598 .last()
17599 .unwrap()
17600 .finished
17601 );
17602 assert_eq!(runner.state.base_episode_used(), 1);
17603 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17604 assert!(!runner.state.base_sync_blocked());
17605 }
17606
17607 #[cfg(unix)]
17608 #[tokio::test]
17609 async fn base_recovery_crash_preserves_untracked_files() {
17610 for ignored in [false, true] {
17611 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17612 runner.state.authorize_base_sync(&task).unwrap();
17613 assert!(runner.state.consume_base_recovery(&head));
17614 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17615 runner.state.reserve_base_operation(&head, &tip);
17616 runner.state.save().unwrap();
17617 let scratch = runner.state.dir().join("base-sync");
17618 assert_eq!(
17619 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17620 .await
17621 .unwrap(),
17622 crate::rebase::Rebased::Applied
17623 );
17624 let repo = &runner.state.repo;
17625 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17626 assert_ne!(result, head);
17627 if ignored {
17628 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17629 .await
17630 .unwrap();
17631 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17632 }
17633 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17634 assert!(
17635 git::git_raw(repo, &["diff", "--quiet", &head])
17636 .await
17637 .unwrap()
17638 .ok()
17639 );
17640 runner.state = RunState::load(&runner.state.id).unwrap();
17642 runner.sync_to_base().await.unwrap();
17643 assert!(runner.state.base_sync_blocked());
17644 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17645 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17646 assert_eq!(
17647 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17648 result
17649 );
17650 assert_eq!(
17651 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17652 "retained work\n"
17653 );
17654 assert!(
17655 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17656 .await
17657 .unwrap()
17658 .ok()
17659 );
17660 }
17661 }
17662
17663 #[cfg(unix)]
17664 #[tokio::test]
17665 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17666 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17667 runner.state.authorize_base_sync(&task).unwrap();
17668 assert!(runner.state.consume_base_recovery(&head));
17669 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17670 runner.state.reserve_base_operation(&head, &tip);
17671 runner.state.save().unwrap();
17672 let scratch = runner.state.dir().join("base-sync");
17673 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17674 .await
17675 .unwrap();
17676 git::sync_to_head(&runner.state.repo).await.unwrap();
17677 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17678 git::git(&runner.state.repo, &["add", "foreign"])
17679 .await
17680 .unwrap();
17681 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17682 .await
17683 .unwrap();
17684 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17685 runner.state = RunState::load(&runner.state.id).unwrap();
17686 runner.sync_to_base().await.unwrap();
17687 assert!(runner.state.base_sync_blocked());
17688 assert!(
17689 !runner
17690 .state
17691 .base_sync_recovery
17692 .operations
17693 .last()
17694 .unwrap()
17695 .finished
17696 );
17697 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17698 assert_eq!(
17699 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17700 foreign_head
17701 );
17702 }
17703
17704 #[cfg(unix)]
17705 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17706 {
17707 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17708 fresh_base_interval_fixture(&mut runner);
17709 let repo = &runner.state.repo;
17710 git::git(repo, &["checkout", "main"]).await.unwrap();
17711 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17712 git::git(repo, &["add", "retained"]).await.unwrap();
17713 git::git(repo, &["commit", "-m", "upstream retained change"])
17714 .await
17715 .unwrap();
17716 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17717 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17718 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17719 let scratch = runner.state.dir().join("base-sync");
17720 assert!(matches!(
17721 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17722 .await
17723 .unwrap(),
17724 git::RebaseStart::Conflicted(_)
17725 ));
17726 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17727 agent: "fixture".into(),
17728 paths: vec!["retained".into()],
17729 from: Some(head.clone()),
17730 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17731 index_outside: None,
17732 finished: false,
17733 error: None,
17734 launcher: None,
17735 infra: None,
17736 outage: None,
17737 });
17738 (tmp, runner, head, tip, scratch)
17739 }
17740
17741 #[cfg(unix)]
17742 #[tokio::test]
17743 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17744 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17745 std::fs::write(
17748 scratch.join("retained"),
17749 "upstream and retained resolution\n",
17750 )
17751 .unwrap();
17752 runner.state.resource_block = Some(crate::run::ResourceBlock {
17753 node: "rebase".into(),
17754 kind: "repository coordination".into(),
17755 detail: "busy".into(),
17756 at: Timestamp::now(),
17757 cause: crate::run::ResourceBlockCause::Other,
17758 });
17759 runner.state.save().unwrap();
17760 let newer = git::git(
17763 &runner.state.repo,
17764 &[
17765 "commit-tree",
17766 "main^{tree}",
17767 "-p",
17768 "main",
17769 "-m",
17770 "base advanced during wait",
17771 ],
17772 )
17773 .await
17774 .unwrap();
17775 git::git(
17776 &runner.state.repo,
17777 &[
17778 "push",
17779 "origin",
17780 &format!("{}:refs/heads/main", newer.trim()),
17781 ],
17782 )
17783 .await
17784 .unwrap();
17785 runner.state = RunState::load(&runner.state.id).unwrap();
17786 runner.state.resource_block = None;
17787 runner.sync_to_base().await.unwrap();
17788 assert!(!runner.state.base_sync_blocked());
17789 let op = &runner.state.base_sync_recovery.operations[0];
17790 assert_eq!(op.head, head);
17791 assert_eq!(op.tip, tip);
17792 assert!(op.finished);
17793 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17794 assert_eq!(runner.state.rebase_fixes.len(), 1);
17795 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17796 .await
17797 .unwrap();
17798 assert_ne!(latest, tip);
17799 runner.state = RunState::load(&runner.state.id).unwrap();
17800 runner.sync_to_base().await.unwrap();
17801 assert!(!runner.state.base_sync_blocked());
17802 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17803 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17804 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17805 }
17806
17807 #[cfg(unix)]
17808 #[tokio::test]
17809 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17810 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17811 runner.state.reserve_base_operation(&head, &tip);
17812 runner.state.save().unwrap();
17813 std::fs::write(
17814 scratch.join("retained"),
17815 "upstream and retained resolution\n",
17816 )
17817 .unwrap();
17818 git::git(&scratch, &["add", "retained"]).await.unwrap();
17819 git::git(
17820 &scratch,
17821 &["-c", "core.editor=true", "rebase", "--continue"],
17822 )
17823 .await
17824 .unwrap();
17825 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17826 assert!(
17827 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17828 .await
17829 .unwrap()
17830 .is_empty()
17831 );
17832 assert_eq!(
17833 crate::rebase::finish(
17834 &mut runner.state,
17835 &scratch,
17836 "fix/retained",
17837 &head,
17838 &tip,
17839 &["retained".into()],
17840 ""
17841 )
17842 .await
17843 .unwrap(),
17844 crate::rebase::Rebased::Applied
17845 );
17846 assert!(!scratch.exists());
17847 runner.state = RunState::load(&runner.state.id).unwrap();
17849 let op = &runner.state.base_sync_recovery.operations[0];
17850 assert!(!op.finished);
17851 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17852 runner.sync_to_base().await.unwrap();
17853 assert!(!runner.state.base_sync_blocked());
17854 assert!(runner.state.base_sync_recovery.operations[0].finished);
17855 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17856 assert_eq!(runner.state.rebase_fixes.len(), 1);
17857 let winner = runner.state.winner().unwrap().clone();
17858 let wait = runner.repository_wait();
17859 assert!(
17860 verify_current_head(&mut runner.state, &winner, &wait)
17861 .await
17862 .unwrap()
17863 );
17864 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17865 }
17866
17867 #[cfg(unix)]
17868 #[tokio::test]
17869 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17870 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17871 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17872 runner.state.reserve_base_operation(&head, &tip);
17873 runner.state.save().unwrap();
17874 std::fs::write(
17875 scratch.join("retained"),
17876 "upstream and retained resolution\n",
17877 )
17878 .unwrap();
17879 git::git(&scratch, &["add", "retained"]).await.unwrap();
17880 git::git(
17881 &scratch,
17882 &["-c", "core.editor=true", "rebase", "--continue"],
17883 )
17884 .await
17885 .unwrap();
17886 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17887 assert!(!git::rebase_in_progress(&scratch).await);
17888 assert!(
17889 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17890 .await
17891 .unwrap()
17892 .is_empty()
17893 );
17894 match damage {
17895 Some("record") => {
17896 let path =
17897 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17898 .await
17899 .unwrap();
17900 std::fs::write(scratch.join(path), "").unwrap();
17901 }
17902 Some("foreign") => {
17903 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17904 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17905 git::git(&scratch, &["commit", "-m", "foreign work"])
17906 .await
17907 .unwrap();
17908 }
17909 Some("dirty") => {
17910 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17911 }
17912 _ => {}
17913 }
17914 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17915 runner.state = RunState::load(&runner.state.id).unwrap();
17917 assert!(
17918 runner.state.base_sync_recovery.operations[0]
17919 .validated_head
17920 .is_none()
17921 );
17922 runner.sync_to_base().await.unwrap();
17923 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17924 assert_eq!(runner.state.base_episode_used(), 1);
17925 assert_eq!(runner.state.rebase_fixes.len(), 1);
17926 assert_eq!(
17927 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17928 retained_head
17929 );
17930 if damage.is_some() {
17931 assert!(runner.state.base_sync_blocked());
17932 assert!(scratch.exists());
17933 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17934 assert!(
17935 runner.state.base_sync_recovery.operations[0]
17936 .validated_head
17937 .is_none()
17938 );
17939 } else {
17940 assert!(!runner.state.base_sync_blocked());
17941 assert!(!scratch.exists());
17942 assert!(runner.state.base_sync_recovery.operations[0].finished);
17943 assert_eq!(
17944 runner.state.base_sync_recovery.operations[0]
17945 .validated_head
17946 .as_deref(),
17947 Some(result.as_str())
17948 );
17949 let winner = runner.state.winner().unwrap().clone();
17950 let wait = runner.repository_wait();
17951 assert!(
17952 verify_current_head(&mut runner.state, &winner, &wait)
17953 .await
17954 .unwrap()
17955 );
17956 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17957 runner.state = RunState::load(&runner.state.id).unwrap();
17958 runner.sync_to_base().await.unwrap();
17959 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17960 }
17961 }
17962 }
17963
17964 #[cfg(unix)]
17965 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17966 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17967 runner.state.prep_complete = Some(true);
17969 runner.state.base_sync = None;
17970 runner.state.base_sync_recovery = Default::default();
17971 runner.state.config.graph.gate_fix_rounds = 1;
17972 let mut review = review_round(true, 0, 1, 1, false, true);
17973 review.head = head.clone();
17974 review.verified_head = Some(head.clone());
17975 runner.state.reviews = vec![review];
17976 runner.state.gate = vec![CommandOutcome {
17977 command: "old gate".into(),
17978 code: Some(1),
17979 output_tail: "old failure".into(),
17980 duration_ms: 0,
17981 resource_blocked: false,
17982 }];
17983 runner.state.gate_head = Some(head.clone());
17984 runner.state.gate_ran = true;
17985 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17986 agent: "fixture".into(),
17987 failed: runner.state.gate.clone(),
17988 notes: "spent".into(),
17989 committed: false,
17990 error: None,
17991 }];
17992 runner.state.land_approval = Some(crate::run::LandApproval {
17993 question: "old approval".into(),
17994 head: head.clone(),
17995 });
17996 runner.state.status = RunStatus::Blocked;
17997 (tmp, runner, head)
17998 }
17999
18000 #[cfg(unix)]
18001 #[tokio::test]
18002 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18003 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18004 let (tmp, mut runner, old) = resumed_gate_fixture().await;
18005 let phases = tmp.path().join("phases");
18006 runner.state.config.verify.e2e = vec![format!(
18007 "echo e2e >> '{}'; {}",
18008 phases.display(),
18009 if e2e_ok {
18010 "true"
18011 } else {
18012 "echo new-e2e-failure; false"
18013 }
18014 )];
18015 runner.state.config.verify.gate = vec![format!(
18016 "echo gate >> '{}'; {}",
18017 phases.display(),
18018 if gate_ok {
18019 "true"
18020 } else {
18021 "echo new-gate-failure; false"
18022 }
18023 )];
18024 runner.state.save().unwrap();
18025 runner.state = RunState::load(&runner.state.id).unwrap();
18026 runner.sync_to_base().await.unwrap();
18027 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18028 assert_ne!(head, old);
18029 runner.state = RunState::load(&runner.state.id).unwrap();
18031 runner.execute_graph().await.unwrap();
18032 assert_eq!(runner.state.gate_fixes.len(), 1);
18033 assert_eq!(runner.state.reviews.len(), 1);
18034 assert_eq!(runner.state.reviews[0].head, old);
18035 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18036 assert_eq!(runner.state.verification_history.len(), 1);
18037 let history = &runner.state.verification_history[0];
18038 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18039 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18040 assert!(history.archived_at.is_some());
18041 assert_eq!(history.fixes_spent, Some(1));
18042 assert_eq!(history.gate[0].output_tail, "old failure");
18043 let archive_events: Vec<_> = runner
18044 .state
18045 .events
18046 .iter()
18047 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18048 .collect();
18049 assert_eq!(archive_events.len(), 1);
18050 assert!(archive_events[0].message.contains(&short(&old)));
18051 assert!(archive_events[0].message.contains(&short(&head)));
18052 assert!(
18053 archive_events[0]
18054 .message
18055 .contains("1 gate-fix round(s) already spent stay spent")
18056 );
18057 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18058 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18059 if e2e_ok {
18060 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18061 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18062 } else {
18063 assert!(!runner.state.gate_ran);
18064 }
18065 assert_eq!(
18066 runner.state.status,
18067 if e2e_ok && gate_ok {
18068 RunStatus::Ready
18069 } else {
18070 RunStatus::Blocked
18071 }
18072 );
18073 if !e2e_ok || !gate_ok {
18074 let cause = runner.state.verification_blocking_cause().unwrap();
18075 assert!(cause.contains(&head));
18076 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18077 runner.state.event("land", "rebased successfully");
18079 let notice = crate::notices::run_ended(&runner.state).unwrap();
18080 assert!(!format!("{notice:?}").contains("rebased successfully"));
18081 assert!(runner.state.merge.is_none());
18082 }
18083 let before = std::fs::read_to_string(&phases).unwrap();
18084 runner.state.save().unwrap();
18085 runner.state = RunState::load(&runner.state.id).unwrap();
18086 runner.review_loop().await.unwrap();
18087 runner.gate().await.unwrap();
18088 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18089 assert_eq!(runner.state.verification_history.len(), 1);
18090 let history = &runner.state.verification_history[0];
18091 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18092 assert!(history.archived_at.is_some());
18093 assert_eq!(history.fixes_spent, Some(1));
18094 }
18095 }
18096
18097 #[cfg(unix)]
18098 #[tokio::test]
18099 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18100 let (tmp, mut runner, head) = resumed_gate_fixture().await;
18101 let marker = tmp.path().join("gate-ran");
18102 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18103 runner.review_loop().await.unwrap();
18104 runner.gate().await.unwrap();
18105 assert_eq!(runner.state.status, RunStatus::Blocked);
18106 assert!(!marker.exists());
18107 assert!(runner.state.verification_history.is_empty());
18108 runner.state.gate_head = None;
18109 runner.gate().await.unwrap();
18110 assert!(marker.exists());
18111 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18112 assert_eq!(runner.state.verification_history.len(), 1);
18113 assert_eq!(runner.state.gate_fixes.len(), 1);
18114 }
18115
18116 #[cfg(unix)]
18117 #[tokio::test]
18118 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18119 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18120 runner.sync_to_base().await.unwrap();
18121 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18122 runner.state.reconcile_verification_head(&head);
18123 for outcomes in [
18125 Vec::new(),
18126 vec![CommandOutcome {
18127 command: "true".into(),
18128 code: None,
18129 output_tail: "cache unavailable".into(),
18130 duration_ms: 0,
18131 resource_blocked: true,
18132 }],
18133 ] {
18134 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18135 head: head.clone(),
18136 passed: false,
18137 outcomes,
18138 });
18139 runner.state.gate_ran = false;
18140 runner.state.gate.clear();
18141 runner.state.save().unwrap();
18142 runner.state = RunState::load(&runner.state.id).unwrap();
18143 runner.merge().await.unwrap();
18144 assert!(runner.state.merge.is_none());
18145 runner.review_loop().await.unwrap();
18146 runner.gate().await.unwrap();
18147 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18148 assert!(runner.state.gate_allows_progress());
18149 assert_eq!(runner.state.gate_fixes.len(), 1);
18150 assert!(runner.state.merge.is_none());
18151 }
18152 git::git(
18154 &runner.state.repo,
18155 &["commit", "--allow-empty", "-m", "external update"],
18156 )
18157 .await
18158 .unwrap();
18159 runner.merge().await.unwrap();
18160 assert!(runner.state.merge.is_none());
18161 assert_eq!(runner.state.status, RunStatus::Blocked);
18162 }
18163
18164 #[cfg(unix)]
18165 #[tokio::test]
18166 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18167 for waiting_on_gate in [false, true] {
18168 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18169 runner.sync_to_base().await.unwrap();
18170 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18171 runner.state.reconcile_verification_head(&head);
18172 if waiting_on_gate {
18173 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18174 head: head.clone(),
18175 passed: true,
18176 outcomes: Vec::new(),
18177 });
18178 }
18179 let cache = tmp.path().join("cache");
18180 runner.state.config.verify.e2e =
18181 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18182 runner.state.config.verify.gate = vec!["true".into()];
18183 runner.state.config.graph.timeout_resource_wait = 1;
18184 let owner =
18185 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18186 let guard =
18187 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18188 crate::cache::AcquireOutcome::Acquired(g) => g,
18189 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18190 };
18191 runner.review_loop().await.unwrap();
18192 runner.gate().await.unwrap();
18193 assert!(!runner.state.gate_ran);
18194 assert_eq!(runner.state.gate_fixes.len(), 1);
18195 assert!(runner.state.resource_block.is_some());
18196 runner.merge().await.unwrap();
18197 assert!(runner.state.merge.is_none());
18198 drop(guard);
18199 runner.state = RunState::load(&runner.state.id).unwrap();
18200 runner.state.resource_block = None;
18202 runner.review_loop().await.unwrap();
18203 runner.gate().await.unwrap();
18204 assert!(runner.state.gate_allows_progress());
18205 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18206 assert_eq!(runner.state.gate_fixes.len(), 1);
18207 }
18208 }
18209
18210 #[cfg(unix)]
18211 #[tokio::test]
18212 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18213 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18214 runner.sync_to_base().await.unwrap();
18215 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18216 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18217 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18219 runner.review_loop().await.unwrap();
18220 runner.gate().await.unwrap();
18221 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18222 assert_ne!(before, after);
18223 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18224 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18225 assert!(!runner.state.gate_allows_progress());
18226 runner.merge().await.unwrap();
18227 assert!(runner.state.merge.is_none());
18228 assert!(
18229 runner
18230 .state
18231 .verification_history
18232 .iter()
18233 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18234 );
18235
18236 runner.state.reviews[0].clean = false;
18238 runner.state.reviews[0].answered = 0;
18239 runner.state.config.verify.e2e = vec!["true".into()];
18240 runner.review_loop().await.unwrap();
18241 runner.gate().await.unwrap();
18242 runner.merge().await.unwrap();
18243 assert_eq!(runner.state.status, RunStatus::Blocked);
18244 assert!(runner.state.merge.is_none());
18245 assert!(
18246 runner
18247 .state
18248 .verification_blocking_cause()
18249 .unwrap()
18250 .contains("review blocked")
18251 );
18252 }
18253
18254 #[cfg(unix)]
18255 #[tokio::test]
18256 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18257 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18258 runner.state.gate_head = None;
18259 runner.state.config.verify.gate =
18260 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18261 runner.review_loop().await.unwrap();
18262 runner.gate().await.unwrap();
18263 assert_ne!(
18264 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18265 old
18266 );
18267 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18268 assert!(!runner.state.gate_allows_progress());
18269 assert_eq!(runner.state.status, RunStatus::Blocked);
18270 runner.merge().await.unwrap();
18271 assert!(runner.state.merge.is_none());
18272 }
18273
18274 #[cfg(unix)]
18275 #[tokio::test]
18276 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18277 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18278 let mut old_review = review_round(true, 0, 1, 1, false, true);
18279 old_review.head = head.clone();
18280 old_review.verified_head = Some(head.clone());
18281 runner.state.reviews = vec![old_review];
18282 runner.state.gate = vec![CommandOutcome {
18283 command: "old gate".into(),
18284 code: None,
18285 output_tail: "timed out after 2400s".into(),
18286 duration_ms: 2_400_000,
18287 resource_blocked: false,
18288 }];
18289 runner.state.gate_ran = true;
18290 runner.state.gate_head = Some(head.clone());
18291 runner.state.land_approval = Some(crate::run::LandApproval {
18292 question: "old-head-approval".into(),
18293 head: head.clone(),
18294 });
18295 runner.state.authorize_base_sync(&task).unwrap();
18296 runner.sync_to_base().await.unwrap();
18297 assert!(!runner.state.base_sync_blocked());
18298 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18299 assert_ne!(head, fresh);
18300 assert_eq!(runner.state.reviews.len(), 1);
18301 assert!(!runner.state.gate_ran);
18302 let saved = &runner.state.base_sync_recovery.authorizations[0];
18303 assert_eq!(saved.reviews[0].head, head);
18304 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18305 assert!(saved.gate_ran);
18306 assert!(!saved.gate[0].ok());
18307 runner.review_loop().await.unwrap();
18308 runner.gate().await.unwrap();
18309 assert_eq!(runner.state.reviews[0].head, head);
18310 assert_eq!(
18311 runner.state.reviews[0].verified_head.as_deref(),
18312 Some(head.as_str())
18313 );
18314 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18315 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18316 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18317 assert!(runner.state.gate_ran);
18318 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18319 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18320 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18321 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18322 assert_eq!(
18323 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18324 "retained change"
18325 );
18326 }
18327
18328 #[cfg(unix)]
18329 #[tokio::test]
18330 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18331 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18332 let queue = crate::queue::Queue::open();
18333 let mut owner = queue.get(&task).unwrap();
18334 owner.release_operator("test-resume", None);
18335 queue.put(&mut owner).unwrap();
18336 let generation = owner.resumption.as_ref().unwrap().generation;
18337 runner.state.task_instruction = Some(owner.instruction.clone());
18338 runner.state.driver_exited = true;
18339 runner.state.save().unwrap();
18340 let home = crate::run::home();
18341 let _claim = queue.claim(&task).unwrap();
18342 owner.resumption.as_mut().unwrap().at =
18343 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18344 queue.put(&mut owner).unwrap();
18345 assert!(
18346 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18347 .await
18348 .is_err(),
18349 "an approval from before the stop grants nothing"
18350 );
18351 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18352 queue.put(&mut owner).unwrap();
18353 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18354 .await
18355 .unwrap();
18356 assert!(
18357 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18358 .await
18359 .is_err()
18360 );
18361 runner.state = RunState::load(&runner.state.id).unwrap();
18362 assert_eq!(
18363 runner.state.base_sync_recovery.authorizations[0].generation,
18364 Some(generation)
18365 );
18366 runner.sync_to_base().await.unwrap();
18367 let repo = runner.state.repo.clone();
18368 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18369 git::git(&repo, &["checkout", "main"]).await.unwrap();
18370 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18371 git::git(&repo, &["add", "."]).await.unwrap();
18372 git::git(&repo, &["commit", "-m", "move base again"])
18373 .await
18374 .unwrap();
18375 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18376 git::git(&repo, &["checkout", "fix/retained"])
18377 .await
18378 .unwrap();
18379 runner.sync_to_base().await.unwrap();
18380 assert!(!runner.state.base_sync_blocked());
18381 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18382 assert_eq!(runner.state.base_episode_used(), 2);
18383 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18384 assert_ne!(fresh, head);
18385 assert!(!runner.state.base_recovery_pending());
18386 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18388 queue.put(&mut owner).unwrap();
18389 assert!(
18390 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18391 .await
18392 .is_err()
18393 );
18394 runner.state = RunState::load(&runner.state.id).unwrap();
18395 runner.sync_to_base().await.unwrap();
18396 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18397 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18398 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18399 }
18400
18401 #[cfg(unix)]
18402 #[tokio::test]
18403 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18404 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18405 runner.state.base_sync = None;
18406 runner.state.status = RunStatus::Blocked;
18407 let mut second = runner.state.candidates[0].clone();
18408 second.label = 'B';
18409 runner.state.candidates.push(second);
18410 assert!(runner.state.votes.is_empty());
18411 assert!(runner.state.deliberation.is_empty());
18412 runner.deliberate().await.unwrap();
18413 runner.vote().await.unwrap();
18414 assert_eq!(runner.state.status, RunStatus::Blocked);
18415 assert!(runner.state.votes.is_empty());
18416 assert!(runner.state.deliberation.is_empty());
18417 }
18418
18419 fn runner_at(status: RunStatus) -> Runner {
18422 let mut state = RunState::new(
18423 PathBuf::from("/nonexistent/repo"),
18424 "main".to_owned(),
18425 "deadbeef".to_owned(),
18426 "task".to_owned(),
18427 Config::default(),
18428 );
18429 state.config.disk.min_free_bytes = 0;
18431 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18434 state.id = format!(
18435 "20000101-000000-{:04x}",
18436 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18437 );
18438 state.status = status;
18439 Runner {
18440 state,
18441 roles: ResolvedRoles {
18442 implementers: Vec::new(),
18443 judges: Vec::new(),
18444 reviewers: Vec::new(),
18445 fixer: None,
18446 conductor: conductor(),
18447 implementer_roster: Vec::new(),
18448 judge_roster: Vec::new(),
18449 reviewer_roster: Vec::new(),
18450 },
18451 sem: Arc::new(Semaphore::new(1)),
18452 pause: Pause::new(),
18453 interrupt: Pause::new(),
18454 }
18455 }
18456
18457 #[tokio::test]
18458 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18459 crate::run::pin_test_home();
18460 let temp = tempfile::tempdir().unwrap();
18461 let root = temp.path().canonicalize().unwrap();
18462 let repo = root.join("repo");
18463 std::fs::create_dir(&repo).unwrap();
18464 init_repo(&repo);
18465 git::git(&repo, &["config", "core.autocrlf", "false"])
18466 .await
18467 .unwrap();
18468 let mut runner = runner_at(RunStatus::Blocked);
18469 runner.state.repo = repo.clone();
18470 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18471 runner.state.config.agents = vec![conductor()];
18472 runner.state.config.graph.selected_implementer = Some("conductor".into());
18473 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18474 let (shell, command, setup_output) = if cfg!(windows) {
18475 (
18476 vec!["cmd".into(), "/C".into()],
18477 "echo ready>seat-ready",
18478 "ready\r\n",
18479 )
18480 } else {
18481 (
18482 vec!["sh".into(), "-c".into()],
18483 "printf ready > seat-ready",
18484 "ready",
18485 )
18486 };
18487 runner.state.config.verify.shell = Some(shell);
18488 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18489 run: Some(command.into()),
18490 ..Default::default()
18491 }];
18492 runner.state.save().unwrap();
18493 let path = runner.state.worktree_root().join("judge-1");
18494 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18495 .await
18496 .unwrap();
18497 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18498 git::git(&path, &["add", "."]).await.unwrap();
18499 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18500 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18501 .await
18502 .unwrap();
18503 git::git(&path, &["checkout", "--detach", &head])
18504 .await
18505 .unwrap();
18506 let evidence = crate::fold_evidence::snapshot(
18507 &runner.state,
18508 &crate::run::home().canonicalize().unwrap(),
18509 &head,
18510 64 * 1024 * 1024,
18511 )
18512 .await
18513 .unwrap();
18514 let mut contents = BTreeMap::new();
18515 contents.insert(
18516 repo.join("logs/HEAD"),
18517 format!("{head} {head} fixture\n").into_bytes(),
18518 );
18519 let record = serde_json::json!({
18520 "version":1, "run":runner.state.id, "source":path, "head":head,
18521 "evidence":evidence, "admin":{}, "admin_contents":contents,
18522 "history":{head.clone():evidence}, "phase":"evicted"
18523 });
18524 std::fs::write(
18525 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18526 serde_json::to_vec(&record).unwrap(),
18527 )
18528 .unwrap();
18529 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18530 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18531 .await
18532 .unwrap();
18533 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18534 assert!(
18535 !git::git_raw(&repo, &["cat-file", "-e", &head])
18536 .await
18537 .unwrap()
18538 .ok()
18539 );
18540 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18541 let error = resumed.execute().await.unwrap_err();
18542 assert!(
18543 error
18544 .to_string()
18545 .contains("selected implementation is held"),
18546 "{error:#}"
18547 );
18548 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18549 assert_eq!(
18550 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18551 setup_output
18552 );
18553 assert_eq!(
18554 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18555 "unrelated seat"
18556 );
18557 let error = resumed.execute().await.unwrap_err();
18558 assert!(
18559 error
18560 .to_string()
18561 .contains("selected implementation is held"),
18562 "{error:#}"
18563 );
18564 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18565 git::git(&path, &["add", "."]).await.unwrap();
18566 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18567 let next = git::git(
18568 &path,
18569 &[
18570 "commit-tree",
18571 &next_tree,
18572 "-p",
18573 &head,
18574 "-m",
18575 "next review checkpoint",
18576 ],
18577 )
18578 .await
18579 .unwrap();
18580 git::git(&path, &["checkout", "--detach", &next])
18581 .await
18582 .unwrap();
18583 resumed.state.reviews.push(
18584 serde_json::from_value(serde_json::json!({
18585 "round":1, "head":next, "reviews":[]
18586 }))
18587 .unwrap(),
18588 );
18589 resumed.state.save().unwrap();
18590 let error = resumed.execute().await.unwrap_err();
18591 assert!(
18592 error
18593 .to_string()
18594 .contains("selected implementation is held"),
18595 "{error:#}"
18596 );
18597 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18598 }
18599
18600 #[tokio::test]
18601 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18602 crate::run::pin_test_home();
18603 let mut runner = runner_at(RunStatus::Gating);
18604 runner.state.save().unwrap();
18605 let id = runner.state.id.clone();
18606 let passes = runner.state.passes.len();
18607 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18608 let mut claiming = Box::pin(runner.claim_driver());
18609 std::future::poll_fn(|cx| {
18610 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18611 std::task::Poll::Ready(())
18612 })
18613 .await;
18614 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18615 drop(owner);
18616 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18617 .await
18618 .unwrap()
18619 .unwrap();
18620 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18621 assert!(execution.runner.state.resource_block.is_none());
18622 assert_eq!(
18623 execution.runner.state.liveness(false),
18624 crate::run::Liveness::Live
18625 );
18626 drop(execution);
18627 assert!(RunState::load(&id).unwrap().driver_exited);
18628 }
18629
18630 #[tokio::test]
18631 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18632 crate::run::pin_test_home();
18633 let mut runner = runner_at(RunStatus::Gating);
18634 runner.state.save().unwrap();
18635 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18638 assert!(runner.claim_driver().await.is_err());
18639 assert!(runner.state.driver_exited);
18640 assert_eq!(runner.state.driver_pid, None);
18641 let disk = RunState::load(&runner.state.id).unwrap();
18642 assert!(disk.driver_exited);
18643 assert_eq!(disk.driver_pid, None);
18644 assert_eq!(disk.status, RunStatus::Gating);
18645 }
18646
18647 #[tokio::test]
18648 async fn an_early_graph_error_releases_its_driver() {
18649 crate::run::pin_test_home();
18650 let mut runner = runner_at(RunStatus::Blocked);
18651 runner.state.config.graph.selected_implementer = Some("selected".into());
18652 runner.state.save().unwrap();
18653 assert!(runner.execute().await.is_err());
18654 let disk = RunState::load(&runner.state.id).unwrap();
18655 assert!(disk.driver_exited);
18656 assert_eq!(disk.driver_pid, None);
18657 assert_eq!(disk.status, RunStatus::Blocked);
18658 }
18659
18660 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18661 crate::run::MergeRepair {
18662 question: "repair-question".into(),
18663 head: "old-head".into(),
18664 pr: "https://github.com/o/r/pull/42".into(),
18665 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18666 gate: vec![outcome(Some(0), false)],
18667 gate_head: Some("old-head".into()),
18668 gate_fixes: Vec::new(),
18669 pre_gate: Vec::new(),
18670 phase,
18671 failure: None,
18672 }
18673 }
18674
18675 #[tokio::test]
18676 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18677 crate::run::pin_test_home();
18678 let mut runner = runner_at(RunStatus::Landing);
18679 runner.state.merge_repairs.push(merge_repair_request(
18680 crate::run::MergeRepairPhase::Requested,
18681 ));
18682 runner.state.save().unwrap();
18683 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18684 assert!(
18685 runner
18686 .execute()
18687 .await
18688 .unwrap_err()
18689 .to_string()
18690 .contains("already running")
18691 );
18692 assert_eq!(
18693 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18694 crate::run::MergeRepairPhase::Requested
18695 );
18696 drop(claim);
18697 let mut disk = runner.state.clone();
18698 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18699 disk.save().unwrap();
18700 assert!(
18701 runner
18702 .execute()
18703 .await
18704 .unwrap_err()
18705 .to_string()
18706 .contains("decision changed")
18707 );
18708 assert_eq!(
18709 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18710 crate::run::MergeRepairPhase::Complete
18711 );
18712 }
18713
18714 #[test]
18715 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18716 crate::run::pin_test_home();
18717 let mut runner = runner_at(RunStatus::Landing);
18718 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18719 runner.state.reviews = request.reviews.clone();
18720 runner.state.gate = request.gate.clone();
18721 runner.state.gate_ran = true;
18722 runner.state.land_approval = Some(crate::run::LandApproval {
18723 question: "old".into(),
18724 head: "old-head".into(),
18725 });
18726 runner.state.merge_repairs.push(request);
18727 runner.begin_merge_repair_review(0).unwrap();
18728 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18729 assert!(!runner.state.gate_status().ok());
18730 assert!(runner.state.land_approval.is_none());
18731 let disk = RunState::load(&runner.state.id).unwrap();
18732 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18733 assert!(disk.merge_repairs[0].gate[0].ok());
18734 assert_eq!(
18735 disk.merge_repairs[0].phase,
18736 crate::run::MergeRepairPhase::Reviewing
18737 );
18738 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18739 assert!(disk.reviews.is_empty());
18740 }
18741
18742 #[tokio::test]
18743 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18744 crate::run::pin_test_home();
18745 for phase in [
18746 crate::run::MergeRepairPhase::Fixing,
18747 crate::run::MergeRepairPhase::Requested,
18748 ] {
18749 let mut runner = runner_at(RunStatus::Landing);
18750 runner.state.config.graph.land_rounds = 0;
18751 runner.state.merge_repairs.push(merge_repair_request(phase));
18752 runner.state.save().unwrap();
18753 assert!(!runner.repair_merge_review().await.unwrap());
18754 assert_eq!(runner.state.status, RunStatus::Blocked);
18755 assert_eq!(
18756 runner.state.merge_repairs[0].phase,
18757 crate::run::MergeRepairPhase::Failed
18758 );
18759 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18760 assert!(
18761 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18762 "interrupted"
18763 } else {
18764 "budget exhausted"
18765 })
18766 );
18767 assert!(runner.state.merge.is_none());
18768 }
18769 }
18770
18771 #[test]
18775 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18776 crate::run::pin_test_home();
18777 let mut runner = runner_at(RunStatus::Implementing);
18778 let interrupt = Pause::new();
18779 runner.watch_interrupt(interrupt.clone());
18780
18781 interrupt.park_because("task a1b2 asked to run first");
18782
18783 assert!(runner.park_here().expect("park_here"));
18784 assert!(runner.state.parked);
18785 let last = runner.state.events.last().expect("a park event");
18786 assert_eq!(last.node, "park");
18787 assert!(
18788 last.message.contains("task a1b2 asked to run first"),
18789 "expected the interrupt reason in {:?}",
18790 last.message
18791 );
18792 }
18793
18794 #[test]
18797 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18798 crate::run::pin_test_home();
18799 let mut runner = runner_at(RunStatus::Implementing);
18800 runner.state.begin_pass();
18801 let shutdown = Pause::new();
18802 runner.on_pause(shutdown.clone());
18803 let upgrade = ParkCause::Upgrade {
18804 from: "0.121.0".to_owned(),
18805 to: Some("0.121.1".to_owned()),
18806 };
18807 shutdown.park_for(upgrade.clone());
18808 shutdown.park_for(ParkCause::Operator);
18809
18810 assert!(runner.park_here().expect("park_here"));
18811 assert_eq!(
18812 runner.state.passes[0].end,
18813 Some(crate::run::PassEnd::Parked { cause: upgrade })
18814 );
18815 let last = runner.state.events.last().expect("a park event");
18816 assert!(last.message.contains("0.121.0"), "{}", last.message);
18817 }
18818
18819 #[tokio::test]
18820 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18821 crate::run::pin_test_home();
18822 let mut runner = runner_at(RunStatus::Reviewing);
18823 runner.state.save().unwrap();
18824 let mut execution = runner.claim_driver().await.expect("first execution");
18825 assert_eq!(execution.runner.state.passes.len(), 1);
18826 execution
18827 .driver
18828 .finish(&mut execution.runner.state)
18829 .await
18830 .unwrap();
18831 }
18832
18833 #[tokio::test]
18834 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18835 crate::run::pin_test_home();
18836 let mut runner = runner_at(RunStatus::Reviewing);
18837 runner.state.config.agents = vec![conductor()];
18838 runner.state.config.graph.timeout_resource_wait = 0;
18839 runner.state.save().unwrap();
18840 let id = runner.state.id.clone();
18841 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18842 assert!(runner.claim_driver().await.is_err());
18843 drop(owner);
18844 drop(runner);
18845 let mut resumed = Runner::resume(&id).unwrap();
18846 assert_eq!(resumed.state.driver_claimed, Some(false));
18847 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18848 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18849 let mut execution = resumed
18850 .claim_driver()
18851 .await
18852 .expect("retry initial claim after reload");
18853 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18854 assert_eq!(execution.runner.state.passes.len(), 1);
18855 execution
18856 .driver
18857 .finish(&mut execution.runner.state)
18858 .await
18859 .unwrap();
18860 }
18861
18862 #[tokio::test]
18863 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18864 crate::run::pin_test_home();
18865 let mut runner = runner_at(RunStatus::Reviewing);
18866 runner.state.config.agents = vec![conductor()];
18867 runner.state.save().unwrap();
18868 let id = runner.state.id.clone();
18869 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18870 let mut resumed = Runner::resume(&id).unwrap();
18871 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18872 assert!(resumed.claim_driver().await.is_err());
18873 assert!(RunState::load(&id).unwrap().passes.is_empty());
18874 drop(lease);
18875 let mut execution = resumed.claim_driver().await.unwrap();
18876 execution
18877 .driver
18878 .finish(&mut execution.runner.state)
18879 .await
18880 .unwrap();
18881 }
18882
18883 #[tokio::test]
18884 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18885 crate::run::pin_test_home();
18886 let mut runner = runner_at(RunStatus::Reviewing);
18887 let state = &mut runner.state;
18888 state.config.agents = vec![conductor()];
18889 state.driver_claimed = None;
18890 state.driver_pid = Some(std::process::id());
18891 state.driver_started_at = None;
18892 state.begin_pass();
18893 state.save().unwrap();
18894 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18895 std::fs::write(
18896 state.dir().join("driver.json"),
18897 serde_json::to_vec(&serde_json::json!({
18898 "pid": state.driver_pid, "started_at": null,
18899 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18900 }))
18901 .unwrap(),
18902 )
18903 .unwrap();
18904 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18905 drop(lease);
18906 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18907 let mut resumed = Runner::resume(&state.id).unwrap();
18908 assert!(!resumed.state.driver_exited);
18909 let mut execution = resumed
18910 .claim_driver()
18911 .await
18912 .expect("lease proves previous execution ended");
18913 assert_eq!(execution.runner.state.passes.len(), 2);
18914 execution
18915 .driver
18916 .finish(&mut execution.runner.state)
18917 .await
18918 .unwrap();
18919 }
18920
18921 #[tokio::test]
18922 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18923 crate::run::pin_test_home();
18924 let mut runner = runner_at(RunStatus::Reviewing);
18925 let mut json = serde_json::to_value(&runner.state).unwrap();
18926 json.as_object_mut().unwrap().remove("driver_claimed");
18927 runner.state = serde_json::from_value(json).unwrap();
18928 assert_eq!(runner.state.driver_claimed, None);
18929 runner.state.save().unwrap();
18930 let mut execution = runner
18931 .claim_driver()
18932 .await
18933 .expect("the execution lease, not a missing marker, excludes a second driver");
18934 assert_eq!(execution.runner.state.passes.len(), 1);
18935 execution
18936 .driver
18937 .finish(&mut execution.runner.state)
18938 .await
18939 .unwrap();
18940 }
18941
18942 #[tokio::test]
18943 async fn driver_admission_refuses_a_proven_live_driver() {
18944 crate::run::pin_test_home();
18945 let mut runner = runner_at(RunStatus::Reviewing);
18946 let pid = std::process::id();
18947 runner.state.driver_pid = Some(pid);
18948 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18949 assert!(runner.state.driver_started_at.is_some());
18950 runner.state.save().unwrap();
18951 let error = runner
18952 .claim_driver()
18953 .await
18954 .err()
18955 .expect("a live driver must refuse");
18956 assert!(error.to_string().contains("live driver"), "{error:#}");
18957 let saved = RunState::load(&runner.state.id).unwrap();
18958 assert_eq!(saved.driver_pid, Some(pid));
18959 assert_eq!(saved.status, RunStatus::Reviewing);
18960 assert!(saved.passes.is_empty());
18961 }
18962
18963 #[test]
18964 fn parking_resource_halt_persists_the_pass_once() {
18965 crate::run::pin_test_home();
18966 let mut runner = runner_at(RunStatus::Reviewing);
18967 runner.state.begin_pass();
18968 runner.state.resource_block = Some(crate::run::ResourceBlock {
18969 node: "fix".into(),
18970 kind: "shared build cache".into(),
18971 detail: "cancelled".into(),
18972 at: Timestamp::now(),
18973 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18974 });
18975 let cause = ParkCause::Upgrade {
18976 from: "old".into(),
18977 to: Some("new".into()),
18978 };
18979 runner.pause.park_for(cause.clone());
18980 runner.stop_for_resource().unwrap();
18983 assert!(runner.resource_halt().unwrap());
18984 assert!(runner.park_here().unwrap());
18985 let saved = RunState::load(&runner.state.id).unwrap();
18986 assert!(saved.parked);
18987 assert_eq!(saved.status, RunStatus::Reviewing);
18988 assert!(saved.passes[0].ended_at.is_some());
18989 assert_eq!(
18990 saved.passes[0].end,
18991 Some(crate::run::PassEnd::Parked {
18992 cause: cause.clone()
18993 })
18994 );
18995 assert_eq!(
18996 saved.resource_block.unwrap().cause,
18997 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18998 );
18999 assert_eq!(
19000 saved
19001 .events
19002 .iter()
19003 .filter(|event| event.node == "park")
19004 .count(),
19005 1
19006 );
19007 }
19008
19009 #[test]
19017 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19018 crate::run::pin_test_home();
19019 let mut runner = runner_at(RunStatus::Implementing);
19020 let shutdown = Pause::new();
19021 runner.on_pause(shutdown.clone());
19022 let interrupt = Pause::new();
19023 runner.watch_interrupt(interrupt.clone());
19024
19025 assert!(!runner.park_here().expect("park_here"));
19027 assert!(!runner.state.parked);
19028
19029 interrupt.park_because("test");
19031 assert!(!shutdown.parked());
19032 assert!(runner.park_here().expect("park_here"));
19033 }
19034
19035 #[tokio::test]
19049 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19050 crate::run::pin_test_home();
19051 let mut runner = runner_at(RunStatus::Implementing);
19052 let interrupt = Pause::new();
19053 runner.watch_interrupt(interrupt.clone());
19054
19055 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19056 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19057
19058 let node = async move {
19062 started_tx.send(()).expect("send started");
19063 finish_rx.await.expect("recv finish");
19064 "node finished"
19065 };
19066
19067 let interrupter = async move {
19068 started_rx.await.expect("recv started");
19069 interrupt.park_because("higher-priority task waiting");
19071 tokio::task::yield_now().await;
19075 finish_tx.send(()).expect("send finish");
19076 };
19077
19078 let (node_result, ()) = tokio::join!(node, interrupter);
19079 assert_eq!(
19080 node_result, "node finished",
19081 "the in-flight call ran to completion"
19082 );
19083
19084 assert!(runner.park_here().expect("park_here"));
19087 assert!(runner.state.parked);
19088 }
19089
19090 #[test]
19096 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19097 crate::run::pin_test_home();
19098 let mut runner = runner_at(RunStatus::Judging);
19099 runner.state.config.agents = vec![conductor()];
19103 runner.state.candidates = vec![Candidate {
19104 index: 0,
19105 label: 'A',
19106 agent: "alpha".to_owned(),
19107 branch: "magi/x/A".to_owned(),
19108 worktree: PathBuf::from("/nonexistent/worktree"),
19109 summary: "did the thing".to_owned(),
19110 stat: "1 file changed".to_owned(),
19111 files: 1,
19112 commits: 1,
19113 empty: false,
19114 failed: None,
19115 verified_noop: None,
19116 duration_ms: 1234,
19117 folded: false,
19118 }];
19119 let run_id = runner.state.id.clone();
19120
19121 let interrupt = Pause::new();
19122 runner.watch_interrupt(interrupt.clone());
19123 interrupt.park_because("task c3d4 asked to run first");
19124 assert!(runner.park_here().expect("park_here"));
19125
19126 let resumed = Runner::resume(&run_id).expect("resume");
19127 assert_eq!(resumed.state.candidates.len(), 1);
19128 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19129 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19130 assert_eq!(resumed.state.status, runner.state.status);
19131 assert!(
19132 resumed.state.parked,
19133 "still parked until `execute` actually walks the graph again"
19134 );
19135 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19136 }
19137
19138 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19140 let mut q = ask::Question::new(
19141 run.to_owned(),
19142 "implement".to_owned(),
19143 "impl-A".to_owned(),
19144 "Which storage backend should the cache use?".to_owned(),
19145 String::new(),
19146 vec!["SQLite".to_owned(), "Redis".to_owned()],
19147 );
19148 store.put(&mut q).unwrap();
19149 q
19150 }
19151
19152 #[test]
19153 fn a_failed_runs_open_question_is_abandoned() {
19154 ask_test_home();
19155 let store = ask::Questions::open();
19156 let mut runner = runner_at(RunStatus::Failed);
19157 let run = runner.state.id.clone();
19158 let q = ask_open_question(&store, &run);
19159
19160 runner.settle_questions();
19161
19162 let back = store.get(&q.id).unwrap();
19163 assert!(
19164 !back.status.open(),
19165 "the seat that asked died with the run; nobody is left to read an answer"
19166 );
19167 assert!(
19168 back.detail.contains(&run) && back.detail.contains("failed"),
19169 "the reason names what the run became, not just that it is gone: {}",
19170 back.detail
19171 );
19172 }
19173
19174 #[test]
19175 fn a_merged_runs_open_question_is_abandoned_too() {
19176 ask_test_home();
19177 let store = ask::Questions::open();
19178 for status in [RunStatus::Merged, RunStatus::Ready] {
19181 let mut runner = runner_at(status);
19182 let run = runner.state.id.clone();
19183 let q = ask_open_question(&store, &run);
19184
19185 runner.settle_questions();
19186
19187 let back = store.get(&q.id).unwrap();
19188 assert!(
19189 !back.status.open(),
19190 "{status:?} run's question must not outlive the run"
19191 );
19192 }
19193 }
19194
19195 #[test]
19196 fn a_still_resumable_runs_open_question_is_left_alone() {
19197 ask_test_home();
19198 let store = ask::Questions::open();
19199 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19205 let mut runner = runner_at(status);
19206 let run = runner.state.id.clone();
19207 let q = ask_open_question(&store, &run);
19208
19209 runner.settle_questions();
19210
19211 let back = store.get(&q.id).unwrap();
19212 assert!(
19213 back.status.open(),
19214 "{status:?} is still alive; the question must still be waiting"
19215 );
19216 }
19217 }
19218
19219 #[test]
19220 fn settle_questions_never_touches_an_already_answered_question() {
19221 ask_test_home();
19222 let store = ask::Questions::open();
19223 let mut runner = runner_at(RunStatus::Failed);
19224 let run = runner.state.id.clone();
19225 let mut q = ask_open_question(&store, &run);
19226 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19227 .unwrap();
19228 store.put(&mut q).unwrap();
19229
19230 runner.settle_questions();
19235 runner.settle_questions();
19236
19237 let back = store.get(&q.id).unwrap();
19238 assert_eq!(
19239 back.status,
19240 ask::QuestionStatus::Answered,
19241 "a real answer is a decision on record, never overwritten by a sweep"
19242 );
19243 }
19244
19245 #[tokio::test]
19256 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19257 crate::run::pin_test_home();
19258 let tmp = tempfile::tempdir().expect("tempdir");
19259 let repo = tmp.path().join("repo");
19260 std::fs::create_dir_all(&repo).unwrap();
19261 init_repo(&repo);
19262
19263 let mut config = Config::default();
19264 config.graph.worktree_root = Some(tmp.path().join("wt"));
19265
19266 let mut state = RunState::new(
19267 repo.clone(),
19268 "main".to_owned(),
19269 "deadbeef".to_owned(),
19270 "task".to_owned(),
19271 config,
19272 );
19273 let root = state.worktree_root();
19274 let wt_a = root.join("cand-A");
19275 let wt_b = root.join("cand-B");
19276 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19277 .await
19278 .expect("worktree A");
19279 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19280 .await
19281 .expect("worktree B");
19282
19283 state.candidates = vec![
19284 Candidate {
19285 index: 0,
19286 label: 'A',
19287 agent: "alpha".to_owned(),
19288 branch: "magi/x/A".to_owned(),
19289 worktree: wt_a.clone(),
19290 summary: String::new(),
19291 stat: String::new(),
19292 files: 0,
19293 commits: 0,
19294 empty: false,
19295 failed: None,
19296 verified_noop: None,
19297 duration_ms: 0,
19298 folded: false,
19299 },
19300 Candidate {
19301 index: 1,
19302 label: 'B',
19303 agent: "beta".to_owned(),
19304 branch: "magi/x/B".to_owned(),
19305 worktree: wt_b.clone(),
19306 summary: String::new(),
19307 stat: String::new(),
19308 files: 0,
19309 commits: 0,
19310 empty: false,
19311 failed: None,
19312 verified_noop: None,
19313 duration_ms: 0,
19314 folded: false,
19315 },
19316 ];
19317 state.tally = Some(Tally {
19318 first_choice: BTreeMap::from([('A', 1)]),
19319 borda: BTreeMap::new(),
19320 winner: 'A',
19321 rankings: 1,
19322 unanimous_initial: true,
19323 deliberated: false,
19324 changed_votes: 0,
19325 unanimous_final: true,
19326 tie_break: None,
19327 judges: 1,
19328 present: 1,
19329 quorum: 1,
19330 met_quorum: true,
19331 uncontested: None,
19332 });
19333 state.status = RunStatus::Ready;
19334
19335 fold_run(&mut state, false, &crate::run::home())
19336 .await
19337 .expect("fold_run");
19338
19339 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19340 assert!(
19341 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19342 "the unmerged winner's branch survives"
19343 );
19344 assert!(
19345 !state.candidates[0].folded,
19346 "the winner is not marked folded"
19347 );
19348
19349 assert!(!wt_b.exists(), "the loser's worktree is removed");
19350 assert!(
19351 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19352 "the loser's branch is removed"
19353 );
19354 assert!(state.candidates[1].folded, "the loser is marked folded");
19355 }
19356
19357 #[tokio::test]
19358 #[cfg(unix)]
19359 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19360 use std::os::fd::AsRawFd as _;
19361 use std::time::Duration;
19362
19363 let tmp = tempfile::tempdir().unwrap();
19364 let repo = tmp.path().join("repo");
19365 std::fs::create_dir_all(&repo).unwrap();
19366 init_repo(&repo);
19367 let common = repo.join(".git");
19368 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19369 let home = tmp.path().join("home");
19370 let mut config = Config::default();
19371 config.coordination = crate::config::test_coordination();
19372 config.graph.worktree_root = Some(tmp.path().join("wt"));
19373 config.graph.timeout_resource_wait = 1;
19374 let mut state = RunState::new(
19375 repo.clone(),
19376 "main".into(),
19377 "deadbeef".into(),
19378 "task".into(),
19379 config,
19380 );
19381 let root = state.worktree_root();
19382 let candidate = root.join("cand-A");
19383 let reviewer = root.join("review-1");
19384 let branch = state.branch_for('A');
19385 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19386 .await
19387 .unwrap();
19388 git::worktree_add_detached(&repo, &reviewer, "main")
19389 .await
19390 .unwrap();
19391 git::acquire_worktree_config(&repo).await.unwrap();
19392 state.enabled_worktree_config = true;
19393 state.status = RunStatus::Ready;
19394 state.candidates.push(Candidate {
19395 index: 0,
19396 label: 'A',
19397 agent: "implement".into(),
19398 branch: branch.clone(),
19399 worktree: candidate.clone(),
19400 summary: String::new(),
19401 stat: String::new(),
19402 files: 0,
19403 commits: 0,
19404 empty: false,
19405 failed: None,
19406 verified_noop: None,
19407 duration_ms: 0,
19408 folded: false,
19409 });
19410 state.save_under(&home).unwrap();
19411 let config_before = std::fs::read(common.join("config")).unwrap();
19412 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19413 .await
19414 .unwrap();
19415 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19418 assert_eq!(
19419 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19420 0
19421 );
19422 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19423 assert!(
19424 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19425 .await
19426 .is_err()
19427 );
19428 assert!(candidate.exists());
19429 assert!(reviewer.exists());
19430 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19431 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19432 assert_eq!(
19433 git::git(&repo, &["worktree", "list", "--porcelain"])
19434 .await
19435 .unwrap(),
19436 worktrees_before
19437 );
19438 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19439 .await
19440 .unwrap()
19441 .unwrap_err();
19442 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19443 drop(fold);
19444 assert!(candidate.exists() && reviewer.exists());
19445 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19446 assert_eq!(
19447 state.resource_block.as_ref().unwrap().kind,
19448 "repository coordination"
19449 );
19450 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19451 drop(external);
19452 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19453 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19454 .await
19455 .unwrap()
19456 .unwrap();
19457 drop(fold);
19458 assert!(removed.contains(&branch));
19459 assert!(!root.exists());
19460 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19461 assert!(state.candidates[0].folded);
19462 assert!(!state.enabled_worktree_config);
19463 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19464 let enabled = git::git_raw(
19465 &repo,
19466 &["config", "--local", "--get", "extensions.worktreeConfig"],
19467 )
19468 .await
19469 .unwrap();
19470 assert!(!enabled.ok(), "the last config reference was released");
19471 }
19472
19473 #[tokio::test]
19476 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19477 let tmp = tempfile::tempdir().expect("tempdir");
19478 let repo = tmp.path().join("repo");
19479 std::fs::create_dir_all(&repo).unwrap();
19480 init_repo(&repo);
19481 let home = tmp.path().join("home");
19482
19483 let mut config = Config::default();
19484 config.graph.worktree_root = Some(tmp.path().join("wt"));
19485 let mut state = RunState::new(
19486 repo.clone(),
19487 "main".to_owned(),
19488 "deadbeef".to_owned(),
19489 "task".to_owned(),
19490 config,
19491 );
19492 git::git(&repo, &["branch", "magi/x/A", "main"])
19494 .await
19495 .expect("branch");
19496 state.candidates = vec![Candidate {
19497 index: 0,
19498 label: 'A',
19499 agent: "alpha".to_owned(),
19500 branch: "magi/x/A".to_owned(),
19501 worktree: state.worktree_root().join("cand-A"),
19502 summary: String::new(),
19503 stat: String::new(),
19504 files: 0,
19505 commits: 0,
19506 empty: false,
19507 failed: None,
19508 verified_noop: None,
19509 duration_ms: 0,
19510 folded: true,
19511 }];
19512 state.released_to = Some("20260901-000000-new1".to_owned());
19513 state.released_branches = vec!["magi/x/A".to_owned()];
19514
19515 fold_run(&mut state, true, &home).await.expect("fold_run");
19516 state.status = RunStatus::Merged;
19517 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19518 state.merge = Some(crate::run::MergeOutcome {
19519 mode: MergeMode::Pr,
19520 ok: true,
19521 detail: "confirmed merged".into(),
19522 empty: false,
19523 });
19524 fold_run(&mut state, true, &home)
19525 .await
19526 .expect("merged fold");
19527
19528 assert!(
19529 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19530 "the handed-over branch survives a fold"
19531 );
19532 }
19533
19534 #[tokio::test]
19538 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19539 let tmp = tempfile::tempdir().expect("tempdir");
19540 let repo = tmp.path().join("repo");
19541 std::fs::create_dir_all(&repo).unwrap();
19542 init_repo(&repo);
19543 let run = |args: &[&str]| {
19544 let out = std::process::Command::new("git")
19545 .quiet()
19546 .args(args)
19547 .current_dir(&repo)
19548 .output()
19549 .expect("spawn git");
19550 assert!(out.status.success(), "git {args:?}");
19551 };
19552 run(&["branch", "magi/x/A"]);
19553 run(&["checkout", "-q", "-b", "magi/x/B"]);
19554 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19555 run(&["add", "-A"]);
19556 run(&["commit", "-q", "-m", "work"]);
19557 run(&["checkout", "-q", "main"]);
19558
19559 let probe = RunState::new(
19563 repo.clone(),
19564 "main".to_owned(),
19565 "deadbeef".to_owned(),
19566 "task".to_owned(),
19567 Config::default(),
19568 );
19569 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19570 let bare = tmp.path().join("origin.git");
19571 let out = std::process::Command::new("git")
19572 .quiet()
19573 .args(["init", "-q", "--bare"])
19574 .arg(&bare)
19575 .output()
19576 .expect("spawn git");
19577 assert!(out.status.success(), "git init --bare");
19578 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19579 run(&["push", "-q", "origin", "main"]);
19580
19581 let mut state = RunState::new(
19582 repo.clone(),
19583 "main".to_owned(),
19584 "deadbeef".to_owned(),
19585 "task".to_owned(),
19586 Config::default(),
19587 );
19588 state.seeds = vec![refs::Seed {
19589 token: "magi/27b2/A".to_owned(),
19590 kind: refs::SeedKind::Unresolved,
19591 sha: String::new(),
19592 branch: true,
19593 detail: "no branch or commit named magi/27b2/A".to_owned(),
19594 }];
19595
19596 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19597 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19598 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19599 let detail = empty_candidate_detail(&state, "main");
19600 assert!(detail.starts_with("empty candidate"), "{detail}");
19601 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19602 assert!(detail.contains("--start-from"), "{detail}");
19603 }
19604
19605 #[tokio::test]
19614 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19615 let tmp = tempfile::tempdir().expect("tempdir");
19616 let repo = tmp.path().join("repo");
19617 std::fs::create_dir_all(&repo).unwrap();
19618 init_repo(&repo);
19619
19620 let mut config = Config::default();
19621 config.merge.mode = MergeMode::Local;
19622
19623 let mut state = RunState::new(
19624 repo.clone(),
19625 "main".to_owned(),
19626 "deadbeef".to_owned(),
19627 "task".to_owned(),
19628 config,
19629 );
19630 state.candidates = vec![Candidate {
19631 index: 0,
19632 label: 'A',
19633 agent: "alpha".to_owned(),
19634 branch: "does-not-exist".to_owned(),
19635 worktree: repo.clone(),
19636 summary: String::new(),
19637 stat: String::new(),
19638 files: 0,
19639 commits: 0,
19640 empty: false,
19641 failed: None,
19642 verified_noop: None,
19643 duration_ms: 0,
19644 folded: false,
19645 }];
19646 state.tally = Some(Tally {
19647 first_choice: BTreeMap::from([('A', 1)]),
19648 borda: BTreeMap::new(),
19649 winner: 'A',
19650 rankings: 1,
19651 unanimous_initial: true,
19652 deliberated: false,
19653 changed_votes: 0,
19654 unanimous_final: true,
19655 tie_break: None,
19656 judges: 0,
19657 present: 0,
19658 quorum: 0,
19659 met_quorum: true,
19660 uncontested: Some("only candidate A produced a change".to_owned()),
19661 });
19662 state.reviews = vec![ReviewRound {
19663 round: 1,
19664 head: "deadbeef".to_owned(),
19665 verified_head: None,
19666 verified_at: None,
19667 reviews: Vec::new(),
19668 e2e: Vec::new(),
19669 fix: None,
19670 blocking: 0,
19671 answered: 0,
19672 expected: 0,
19673 clean: true,
19674 verify_retried: false,
19675 e2e_deferred: false,
19676 e2e_defer_reason: None,
19677 progressed: false,
19678 vote_split: false,
19679 reconsideration: Vec::new(),
19680 verdict: None,
19681 }];
19682 state.gate = vec![CommandOutcome {
19683 command: "test".to_owned(),
19684 code: Some(0),
19685 output_tail: String::new(),
19686 duration_ms: 0,
19687 resource_blocked: false,
19688 }];
19689 state.gate_ran = true;
19690 state.status = RunStatus::Ready;
19695 state.merge = Some(MergeOutcome {
19696 mode: MergeMode::Local,
19697 ok: false,
19698 detail: "already concluded".to_owned(),
19699 empty: false,
19700 });
19701
19702 let mut runner = Runner {
19703 state,
19704 roles: ResolvedRoles {
19705 implementers: Vec::new(),
19706 judges: Vec::new(),
19707 reviewers: Vec::new(),
19708 fixer: None,
19709 conductor: conductor(),
19710 implementer_roster: Vec::new(),
19711 judge_roster: Vec::new(),
19712 reviewer_roster: Vec::new(),
19713 },
19714 sem: Arc::new(Semaphore::new(1)),
19715 pause: Pause::new(),
19716 interrupt: Pause::new(),
19717 };
19718
19719 runner.merge().await.expect("merge");
19720
19721 assert_eq!(
19722 runner.state.status,
19723 RunStatus::Ready,
19724 "a concluded run's status must not change on reentry"
19725 );
19726 assert_eq!(
19727 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19728 Some("already concluded"),
19729 "merge must not run again once the node already recorded an outcome"
19730 );
19731 }
19732
19733 #[tokio::test]
19742 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19743 let tmp = tempfile::tempdir().expect("tempdir");
19744 let repo = tmp.path().join("repo");
19745 std::fs::create_dir_all(&repo).unwrap();
19746 init_repo(&repo);
19747
19748 let mut config = Config::default();
19749 config.merge.mode = MergeMode::Local;
19750
19751 let mut state = RunState::new(
19752 repo.clone(),
19753 "main".to_owned(),
19754 "deadbeef".to_owned(),
19755 "task".to_owned(),
19756 config,
19757 );
19758 state.candidates = vec![Candidate {
19759 index: 0,
19760 label: 'A',
19761 agent: "alpha".to_owned(),
19762 branch: "does-not-exist".to_owned(),
19763 worktree: repo.clone(),
19764 summary: String::new(),
19765 stat: String::new(),
19766 files: 0,
19767 commits: 0,
19768 empty: false,
19769 failed: None,
19770 verified_noop: None,
19771 duration_ms: 0,
19772 folded: false,
19773 }];
19774 state.tally = Some(Tally {
19775 first_choice: BTreeMap::from([('A', 1)]),
19776 borda: BTreeMap::new(),
19777 winner: 'A',
19778 rankings: 1,
19779 unanimous_initial: true,
19780 deliberated: false,
19781 changed_votes: 0,
19782 unanimous_final: true,
19783 tie_break: None,
19784 judges: 0,
19785 present: 0,
19786 quorum: 0,
19787 met_quorum: true,
19788 uncontested: Some("only candidate A produced a change".to_owned()),
19789 });
19790 state.reviews = vec![ReviewRound {
19791 round: 1,
19792 head: "deadbeef".to_owned(),
19793 verified_head: None,
19794 verified_at: None,
19795 reviews: Vec::new(),
19796 e2e: Vec::new(),
19797 fix: None,
19798 blocking: 0,
19799 answered: 0,
19800 expected: 0,
19801 clean: true,
19802 verify_retried: false,
19803 e2e_deferred: false,
19804 e2e_defer_reason: None,
19805 progressed: false,
19806 vote_split: false,
19807 reconsideration: Vec::new(),
19808 verdict: None,
19809 }];
19810 state.gate = Vec::new();
19812 state.gate_ran = false;
19813 state.status = RunStatus::Gating;
19814
19815 let mut runner = Runner {
19816 state,
19817 roles: ResolvedRoles {
19818 implementers: Vec::new(),
19819 judges: Vec::new(),
19820 reviewers: Vec::new(),
19821 fixer: None,
19822 conductor: conductor(),
19823 implementer_roster: Vec::new(),
19824 judge_roster: Vec::new(),
19825 reviewer_roster: Vec::new(),
19826 },
19827 sem: Arc::new(Semaphore::new(1)),
19828 pause: Pause::new(),
19829 interrupt: Pause::new(),
19830 };
19831
19832 runner.merge().await.expect("merge");
19833
19834 assert!(
19835 runner.state.merge.is_none(),
19836 "an empty gate must never be read as a passing one: {:?}",
19837 runner.state.merge
19838 );
19839 }
19840
19841 #[tokio::test]
19848 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19849 let tmp = tempfile::tempdir().expect("tempdir");
19850 let repo = tmp.path().join("repo");
19851 std::fs::create_dir_all(&repo).unwrap();
19852 init_repo(&repo);
19853
19854 let config = Config::default();
19856
19857 let mut state = RunState::new(
19858 repo.clone(),
19859 "main".to_owned(),
19860 "deadbeef".to_owned(),
19861 "task".to_owned(),
19862 config,
19863 );
19864 state.candidates = vec![Candidate {
19865 index: 0,
19866 label: 'A',
19867 agent: "alpha".to_owned(),
19868 branch: "does-not-exist".to_owned(),
19869 worktree: repo.clone(),
19870 summary: String::new(),
19871 stat: String::new(),
19872 files: 0,
19873 commits: 0,
19874 empty: false,
19875 failed: None,
19876 verified_noop: None,
19877 duration_ms: 0,
19878 folded: false,
19879 }];
19880 state.tally = Some(Tally {
19881 first_choice: BTreeMap::from([('A', 1)]),
19882 borda: BTreeMap::new(),
19883 winner: 'A',
19884 rankings: 1,
19885 unanimous_initial: true,
19886 deliberated: false,
19887 changed_votes: 0,
19888 unanimous_final: true,
19889 tie_break: None,
19890 judges: 0,
19891 present: 0,
19892 quorum: 0,
19893 met_quorum: true,
19894 uncontested: Some("only candidate A produced a change".to_owned()),
19895 });
19896 state.reviews = vec![ReviewRound {
19897 round: 1,
19898 head: "deadbeef".to_owned(),
19899 verified_head: None,
19900 verified_at: None,
19901 reviews: Vec::new(),
19902 e2e: Vec::new(),
19903 fix: None,
19904 blocking: 0,
19905 answered: 0,
19906 expected: 0,
19907 clean: true,
19908 verify_retried: false,
19909 e2e_deferred: false,
19910 e2e_defer_reason: None,
19911 progressed: false,
19912 vote_split: false,
19913 reconsideration: Vec::new(),
19914 verdict: None,
19915 }];
19916
19917 let mut runner = Runner {
19918 state,
19919 roles: ResolvedRoles {
19920 implementers: Vec::new(),
19921 judges: Vec::new(),
19922 reviewers: Vec::new(),
19923 fixer: None,
19924 conductor: conductor(),
19925 implementer_roster: Vec::new(),
19926 judge_roster: Vec::new(),
19927 reviewer_roster: Vec::new(),
19928 },
19929 sem: Arc::new(Semaphore::new(1)),
19930 pause: Pause::new(),
19931 interrupt: Pause::new(),
19932 };
19933
19934 runner.gate().await.expect("gate");
19935 assert!(
19936 runner.state.gate_ran,
19937 "zero configured commands is still a real attempt, not an unrun gate"
19938 );
19939 assert!(runner.state.gate.is_empty());
19940 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19941 assert_ne!(
19942 runner.state.status,
19943 RunStatus::Blocked,
19944 "a gate with nothing to check must not read as failed"
19945 );
19946
19947 runner.merge().await.expect("merge");
19948 assert_eq!(
19949 runner.state.status,
19950 RunStatus::Ready,
19951 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19952 );
19953 }
19954
19955 #[tokio::test]
19966 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19967 crate::run::pin_test_home();
19968 let home = crate::run::home();
19969
19970 let tmp = tempfile::tempdir().expect("tempdir");
19971 let repo = tmp.path().join("repo");
19972 std::fs::create_dir_all(&repo).unwrap();
19973 init_repo(&repo);
19974 let cache_dir = tmp.path().join("target");
19977
19978 let mut config = Config::default();
19979 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19980 config.graph.timeout_verify = Some(2);
19984 config.graph.timeout_resource_wait = 0;
19985
19986 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19987 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19988 .expect("no io error acquiring directly")
19989 {
19990 crate::cache::AcquireOutcome::Acquired(g) => g,
19991 crate::cache::AcquireOutcome::Busy(b) => {
19992 panic!("expected the direct acquire to win the lease first: {b:?}")
19993 }
19994 };
19995
19996 let mut state = RunState::new(
19997 repo.clone(),
19998 "main".to_owned(),
19999 "deadbeef".to_owned(),
20000 "task".to_owned(),
20001 config,
20002 );
20003 state.candidates = vec![Candidate {
20004 index: 0,
20005 label: 'A',
20006 agent: "alpha".to_owned(),
20007 branch: "does-not-exist".to_owned(),
20008 worktree: repo.clone(),
20009 summary: String::new(),
20010 stat: String::new(),
20011 files: 0,
20012 commits: 0,
20013 empty: false,
20014 failed: None,
20015 verified_noop: None,
20016 duration_ms: 0,
20017 folded: false,
20018 }];
20019 state.tally = Some(Tally {
20020 first_choice: BTreeMap::from([('A', 1)]),
20021 borda: BTreeMap::new(),
20022 winner: 'A',
20023 rankings: 1,
20024 unanimous_initial: true,
20025 deliberated: false,
20026 changed_votes: 0,
20027 unanimous_final: true,
20028 tie_break: None,
20029 judges: 0,
20030 present: 0,
20031 quorum: 0,
20032 met_quorum: true,
20033 uncontested: Some("only candidate A produced a change".to_owned()),
20034 });
20035 state.reviews = vec![ReviewRound {
20036 round: 1,
20037 head: "deadbeef".to_owned(),
20038 verified_head: None,
20039 verified_at: None,
20040 reviews: Vec::new(),
20041 e2e: Vec::new(),
20042 fix: None,
20043 blocking: 0,
20044 answered: 0,
20045 expected: 0,
20046 clean: true,
20047 verify_retried: false,
20048 e2e_deferred: false,
20049 e2e_defer_reason: None,
20050 progressed: false,
20051 vote_split: false,
20052 reconsideration: Vec::new(),
20053 verdict: None,
20054 }];
20055
20056 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20057 state.reviews[0].e2e = vec![CommandOutcome {
20058 command: state.config.verify.e2e[0].clone(),
20059 code: Some(0),
20060 output_tail: String::new(),
20061 duration_ms: 0,
20062 resource_blocked: false,
20063 }];
20064 let mut runner = Runner {
20065 state,
20066 roles: ResolvedRoles {
20067 implementers: Vec::new(),
20068 judges: Vec::new(),
20069 reviewers: Vec::new(),
20070 fixer: None,
20071 conductor: conductor(),
20072 implementer_roster: Vec::new(),
20073 judge_roster: Vec::new(),
20074 reviewer_roster: Vec::new(),
20075 },
20076 sem: Arc::new(Semaphore::new(1)),
20077 pause: Pause::new(),
20078 interrupt: Pause::new(),
20079 };
20080
20081 runner.gate().await.expect("gate");
20082 assert!(runner.state.resource_block.is_none());
20083 assert!(
20084 !runner
20085 .state
20086 .events
20087 .iter()
20088 .any(|event| { event.message.contains("waiting for the shared build cache") }),
20089 "a gate with nothing to run must never request the held cache lease"
20090 );
20091 assert!(
20092 runner.state.gate_ran,
20093 "zero commands is still a real, immediate attempt"
20094 );
20095 assert!(runner.state.gate.is_empty());
20096 assert_ne!(
20097 runner.state.status,
20098 RunStatus::Blocked,
20099 "must not read as resource-blocked on a lease it never asked for"
20100 );
20101 }
20102
20103 #[cfg(unix)]
20106 #[tokio::test]
20107 async fn verification_is_standalone_inside_a_live_worker() {
20108 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20109 if std::env::var_os(CHILD).is_none() {
20110 let home = tempfile::tempdir().unwrap();
20111 let generation = crate::worker::pin_current(home.path()).unwrap();
20112 let output = std::process::Command::new(std::env::current_exe().unwrap())
20113 .args([
20114 "--exact",
20115 "graph::tests::verification_is_standalone_inside_a_live_worker",
20116 "--nocapture",
20117 ])
20118 .env(CHILD, "1")
20119 .env("MAGI_HOME", home.path())
20120 .env("MAGI_WORKER_HOME", home.path())
20121 .env("MAGI_WORKER_ID", "abcdef")
20122 .env("MAGI_WORKER_GENERATION", &generation.id)
20123 .env("MAGI_WORKER_EXE", &generation.executable)
20124 .env("MAGI_RUN", "verification-parent")
20125 .env("MAGI_NODE", "gate")
20126 .env("MAGI_SEAT", "worker-fixture")
20127 .env("MAGI_PROCESS_OWNER", "fixture-owner")
20128 .env("MAGI_NO_AUTOUPDATE", "0")
20129 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20130 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20131 .env(
20132 "VERIFY_EXPECT_TOOLCHAIN",
20133 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20134 )
20135 .env(
20136 "VERIFY_EXPECT_CACHE",
20137 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20138 )
20139 .output()
20140 .unwrap();
20141 assert!(
20142 output.status.success()
20143 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20144 "{}\n{}",
20145 String::from_utf8_lossy(&output.stdout),
20146 String::from_utf8_lossy(&output.stderr)
20147 );
20148 return;
20149 }
20150 let home = crate::run::home();
20151 let directory = home.join("workers/jobs/abcdef");
20152 std::fs::create_dir_all(&directory).unwrap();
20153 let job = crate::worker::Job {
20154 id: "abcdef".into(),
20155 sequence: 1,
20156 kind: "task".into(),
20157 key: "task:verification".into(),
20158 payload: serde_json::json!({}),
20159 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20160 version: Some(env!("CARGO_PKG_VERSION").into()),
20161 status: crate::worker::Status::Running,
20162 result: None,
20163 pid: Some(std::process::id()),
20164 created_at: 0,
20165 finished_at: None,
20166 };
20167 std::fs::write(
20168 directory.join("job.json"),
20169 serde_json::to_vec(&job).unwrap(),
20170 )
20171 .unwrap();
20172 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20173 let assert_worker = || {
20174 assert_eq!(
20175 crate::worker::current_job(&home).unwrap().unwrap().id,
20176 job.id
20177 );
20178 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20179 assert_eq!(
20180 crate::worker::helper_executable(&home).unwrap(),
20181 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20182 );
20183 assert!(
20184 crate::updater::Checker::new(&crate::config::Update {
20185 mode: crate::config::UpdateMode::Notify,
20186 interval: None,
20187 auto_restart: false,
20188 })
20189 .is_none(),
20190 "the driver remains pinned, even while tests are standalone"
20191 );
20192 };
20193 assert_worker();
20194 let repo = home.join("repo");
20195 std::fs::create_dir_all(&repo).unwrap();
20196 init_repo(&repo);
20197 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20200 test -z "${MAGI_WORKER_GENERATION+x}" &&
20201 test -z "${MAGI_WORKER_HOME+x}" &&
20202 test -z "${MAGI_WORKER_EXE+x}" &&
20203 test "$MAGI_HOME/repo" -ef . &&
20204 test "$MAGI_RUN" = verification-parent &&
20205 test "$MAGI_NODE" = gate &&
20206 test "$MAGI_SEAT" = worker-fixture &&
20207 test -n "$MAGI_PROCESS_OWNER" &&
20208 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20209 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20210 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20211 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20212 assert!(
20214 !std::process::Command::new("sh")
20215 .args(["-c", probe])
20216 .current_dir(&repo)
20217 .status()
20218 .unwrap()
20219 .success()
20220 );
20221 let unrelated_home = tempfile::tempdir().unwrap();
20222 assert!(
20223 crate::worker::current_job(unrelated_home.path()).is_err(),
20224 "an inherited worker id cannot be resolved in a unit fixture's home"
20225 );
20226 let mut runner = runner_at(RunStatus::Gating);
20227 runner.state.repo = repo.clone();
20228 runner.state.config.disk.min_free_bytes = 0;
20229 runner.state.config.coordination = crate::config::test_coordination();
20230 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20231 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20232 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20233 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20234 let winner = Candidate {
20235 index: 0,
20236 label: 'A',
20237 agent: "fixture".into(),
20238 branch: "main".into(),
20239 worktree: repo.clone(),
20240 summary: String::new(),
20241 stat: String::new(),
20242 files: 0,
20243 commits: 0,
20244 empty: false,
20245 failed: None,
20246 verified_noop: None,
20247 duration_ms: 0,
20248 folded: false,
20249 };
20250 let (e2e, retried, pids) = run_e2e_with_retry(
20251 &mut runner.state,
20252 &["sh".into(), "-c".into()],
20253 &[format!("{probe} && printf initial")],
20254 &repo,
20255 Duration::from_secs(10),
20256 "initial",
20257 )
20258 .await;
20259 assert!(!retried && pids.is_empty());
20260 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20261 assert_eq!(e2e[0].output_tail, "initial");
20262 let gate = runner.run_gate(&winner).await.unwrap();
20263 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20264 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20267 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20268 passed: false,
20269 outcomes: Vec::new(),
20270 });
20271 runner.state.save().unwrap();
20272 let mut resumed = RunState::load(&runner.state.id).unwrap();
20273 let wait = crate::repo_lock::WaitPolicy::default();
20274 assert!(matches!(
20275 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20276 .await
20277 .unwrap(),
20278 GateFix::Retry
20279 ));
20280 let saved = RunState::load(&resumed.id).unwrap();
20281 let repair = saved.gate_repair_e2e.unwrap();
20282 assert!(repair.passed);
20283 assert_eq!(repair.outcomes.len(), 1);
20284 assert!(repair.outcomes[0].ok());
20285 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20286 .await
20287 .unwrap();
20288 assert_eq!(landing.len(), 1);
20289 assert!(landing[0].ok(), "{landing:?}");
20290 assert_eq!(
20291 std::fs::read_to_string(repo.join("phases")).unwrap(),
20292 "pre\ngate\ne2e\npre\ngate\n"
20293 );
20294 assert_worker();
20295 }
20296
20297 #[cfg(unix)]
20298 #[tokio::test]
20299 async fn verification_keeps_the_panic_and_the_full_output() {
20300 crate::run::pin_test_home();
20301 let tmp = tempfile::tempdir().unwrap();
20302 let mut config = Config::default();
20303 config.disk.min_free_bytes = 0;
20304 let mut state = RunState::new(
20305 tmp.path().to_owned(),
20306 "main".into(),
20307 "base".into(),
20308 "verification evidence".into(),
20309 config,
20310 );
20311 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20312 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20313 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20314 let mut names = Vec::new();
20315 for _ in 0..2 {
20316 let (outcomes, _) = run_commands(
20317 &mut state,
20318 "gate",
20319 "gate",
20320 0,
20321 &["sh".into(), "-c".into()],
20322 &[script.into()],
20323 tmp.path(),
20324 Duration::from_secs(60),
20325 )
20326 .await;
20327 let tail = &outcomes[0].output_tail;
20328 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20329 assert!(tail.contains("boom assertion"));
20330 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20331 let line = tail.lines().next().unwrap();
20332 names.push(line.to_owned());
20333 }
20334 assert_ne!(names[0], names[1], "the second run must not overwrite");
20335 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20336 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20337 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20338
20339 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20341 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20342 let (outcomes, _) = run_commands(
20343 &mut state,
20344 "gate",
20345 "gate",
20346 0,
20347 &["sh".into(), "-c".into()],
20348 &["echo failure-output; exit 7".into()],
20349 tmp.path(),
20350 Duration::from_secs(10),
20351 )
20352 .await;
20353 assert_eq!(outcomes[0].code, Some(7));
20354 assert!(state.events.iter().any(|e| {
20355 e.node == "gate" && e.message.contains("could not save the full gate output:")
20356 }));
20357 }
20358
20359 #[cfg(unix)]
20360 #[tokio::test]
20361 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20362 crate::run::pin_test_home();
20363 let tmp = tempfile::tempdir().unwrap();
20364 let mut config = Config::default();
20365 config.disk.min_free_bytes = 0;
20366 let mut state = RunState::new(
20367 tmp.path().to_owned(),
20368 "main".into(),
20369 "base".into(),
20370 "verification evidence".into(),
20371 config,
20372 );
20373 let (outcomes, pids) = run_commands(
20374 &mut state,
20375 "gate",
20376 "gate",
20377 1,
20378 &["sh".into(), "-c".into()],
20379 &[
20380 "printf before-timeout; exec sleep 30".into(),
20381 "kill -TERM $$".into(),
20382 "exit 7".into(),
20383 "printf success".into(),
20384 ],
20385 tmp.path(),
20386 Duration::from_secs(1),
20387 )
20388 .await;
20389 wait_for_timed_out_children_to_die(&pids).await;
20390 assert_eq!(
20391 pids.len(),
20392 1,
20393 "unleased timeout retains its death-check PID"
20394 );
20395 assert_eq!(outcomes.len(), 4);
20396 assert_eq!(outcomes[0].code, None);
20397 assert!(outcomes[0].output_tail.contains("before-timeout"));
20398 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20399 assert_eq!(outcomes[1].code, None);
20400 assert!(outcomes[1].output_tail.contains("command terminated:"));
20401 assert!(!outcomes[1].output_tail.contains("timed out"));
20402 assert_eq!(outcomes[2].code, Some(7));
20403 assert!(!outcomes[2].output_tail.contains("timed out"));
20404 assert_eq!(outcomes[3].code, Some(0));
20405 assert_eq!(outcomes[3].output_tail, "success");
20406 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20407 }
20408
20409 #[tokio::test]
20419 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20420 crate::run::pin_test_home();
20421
20422 let tmp = tempfile::tempdir().expect("tempdir");
20423 let repo = tmp.path().join("repo");
20424 std::fs::create_dir_all(&repo).unwrap();
20425 init_repo(&repo);
20426
20427 let mut config = Config::default();
20428 config.disk.min_free_bytes = 0;
20429 config.verify.gate = vec![
20430 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20431 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20432 .to_owned(),
20433 ];
20434
20435 let mut state = RunState::new(
20436 repo.clone(),
20437 "main".to_owned(),
20438 "deadbeef".to_owned(),
20439 "task".to_owned(),
20440 config,
20441 );
20442 let run_id = state.id.clone();
20443 state.candidates = vec![Candidate {
20444 index: 0,
20445 label: 'A',
20446 agent: "alpha".to_owned(),
20447 branch: "does-not-exist".to_owned(),
20448 worktree: repo.clone(),
20449 summary: String::new(),
20450 stat: String::new(),
20451 files: 0,
20452 commits: 0,
20453 empty: false,
20454 failed: None,
20455 verified_noop: None,
20456 duration_ms: 0,
20457 folded: false,
20458 }];
20459 state.tally = Some(Tally {
20460 first_choice: BTreeMap::from([('A', 1)]),
20461 borda: BTreeMap::new(),
20462 winner: 'A',
20463 rankings: 1,
20464 unanimous_initial: true,
20465 deliberated: false,
20466 changed_votes: 0,
20467 unanimous_final: true,
20468 tie_break: None,
20469 judges: 0,
20470 present: 0,
20471 quorum: 0,
20472 met_quorum: true,
20473 uncontested: Some("only candidate A produced a change".to_owned()),
20474 });
20475 state.reviews = vec![ReviewRound {
20476 round: 1,
20477 head: "deadbeef".to_owned(),
20478 verified_head: None,
20479 verified_at: None,
20480 reviews: Vec::new(),
20481 e2e: Vec::new(),
20482 fix: None,
20483 blocking: 0,
20484 answered: 0,
20485 expected: 0,
20486 clean: true,
20487 verify_retried: false,
20488 e2e_deferred: false,
20489 e2e_defer_reason: None,
20490 progressed: false,
20491 vote_split: false,
20492 reconsideration: Vec::new(),
20493 verdict: None,
20494 }];
20495
20496 let mut runner = Runner {
20497 state,
20498 roles: ResolvedRoles {
20499 implementers: Vec::new(),
20500 judges: Vec::new(),
20501 reviewers: Vec::new(),
20502 fixer: None,
20503 conductor: conductor(),
20504 implementer_roster: Vec::new(),
20505 judge_roster: Vec::new(),
20506 reviewer_roster: Vec::new(),
20507 },
20508 sem: Arc::new(Semaphore::new(1)),
20509 pause: Pause::new(),
20510 interrupt: Pause::new(),
20511 };
20512
20513 let started_marker = repo.join("started.marker");
20514 let release_marker = repo.join("release.marker");
20515 let poller = tokio::spawn(async move {
20516 for _ in 0..100 {
20521 if started_marker.exists()
20522 && let Ok(s) = crate::run::RunState::load(&run_id)
20523 && let Some(a) = s.active.get("gate")
20524 {
20525 std::fs::write(&release_marker, b"go").expect("release marker");
20526 return Some(a.clone());
20527 }
20528 tokio::time::sleep(Duration::from_millis(50)).await;
20529 }
20530 None
20531 });
20532
20533 runner.gate().await.expect("gate");
20534 let captured = poller.await.expect("poller task");
20535 let captured = captured.expect(
20536 "the poller never saw a `gate` task entry in run.json while the command was \
20537 still blocked on its own release marker",
20538 );
20539
20540 assert_eq!(captured.task.as_deref(), Some("gate"));
20541 assert_eq!(captured.node, "gate");
20542 assert_eq!(captured.index, Some(1));
20543 assert_eq!(captured.total, Some(1));
20544 assert!(
20545 captured
20546 .command
20547 .as_deref()
20548 .is_some_and(|c| c.contains("started.marker")),
20549 "{captured:?}"
20550 );
20551
20552 assert!(
20553 runner.state.active.is_empty(),
20554 "the entry must be cleared once the command actually finished: {:?}",
20555 runner.state.active
20556 );
20557 assert!(runner.state.gate_ran);
20558 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20559 }
20560
20561 #[tokio::test]
20564 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20565 use crate::verdict::{Finding, ReviewVote, Severity};
20566 crate::run::pin_test_home();
20567 let tmp = tempfile::tempdir().expect("tempdir");
20568 let repo = tmp.path().join("repo");
20569 std::fs::create_dir_all(&repo).unwrap();
20570 init_repo(&repo);
20571
20572 for (severity, vote, expect) in [
20573 (Severity::Major, ReviewVote::Reject, true),
20574 (Severity::Minor, ReviewVote::Reject, false),
20575 (Severity::Major, ReviewVote::Approve, false),
20576 ] {
20577 let mut round = review_round(false, 1, 1, 1, false, true);
20578 round.reviews = vec![ReviewRecord {
20579 reviewer: 1,
20580 agent: "alpha".to_owned(),
20581 summary: String::new(),
20582 findings: vec![Finding {
20583 id: "R1-1-1".to_owned(),
20584 severity,
20585 file: None,
20586 line: None,
20587 title: "t".to_owned(),
20588 detail: String::new(),
20589 }],
20590 vote: Some(vote),
20591 failed: None,
20592 duration_ms: 0,
20593 attempts: 0,
20594 }];
20595 let mut state = RunState::new(
20596 repo.clone(),
20597 "main".to_owned(),
20598 "deadbeef".to_owned(),
20599 "task".to_owned(),
20600 Config::default(),
20601 );
20602 state.reviews = vec![round];
20603 let mut runner = Runner {
20604 state,
20605 roles: ResolvedRoles {
20606 implementers: Vec::new(),
20607 judges: Vec::new(),
20608 reviewers: Vec::new(),
20609 fixer: None,
20610 conductor: conductor(),
20611 implementer_roster: Vec::new(),
20612 judge_roster: Vec::new(),
20613 reviewer_roster: Vec::new(),
20614 },
20615 sem: Arc::new(Semaphore::new(1)),
20616 pause: Pause::new(),
20617 interrupt: Pause::new(),
20618 };
20619 let shell = runner.state.config.shell();
20620 runner
20621 .stop_reviewing("round budget spent", &shell, &repo)
20622 .await
20623 .expect("stop_reviewing");
20624 assert_eq!(runner.state.status, RunStatus::Gating);
20625 assert_eq!(
20626 runner.state.contested_handoff.is_some(),
20627 expect,
20628 "{severity:?} + {vote:?}"
20629 );
20630 }
20631 }
20632
20633 #[tokio::test]
20646 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20647 crate::run::pin_test_home();
20648 let home = crate::run::home();
20649
20650 let tmp = tempfile::tempdir().expect("tempdir");
20651 let repo = tmp.path().join("repo");
20652 std::fs::create_dir_all(&repo).unwrap();
20653 init_repo(&repo);
20654 let head = crate::git::rev_parse(&repo, "HEAD")
20655 .await
20656 .expect("rev-parse");
20657 let cache_dir = tmp.path().join("target");
20660
20661 let mut config = Config::default();
20662 config.verify.e2e = vec![format!(
20663 "CARGO_TARGET_DIR='{}' test -f README.md",
20664 cache_dir.display()
20665 )];
20666 config.graph.review_rounds = 1;
20667 config.graph.timeout_verify = Some(2);
20670 config.graph.timeout_resource_wait = 2;
20671
20672 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20673 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20674 .expect("no io error acquiring directly")
20675 {
20676 crate::cache::AcquireOutcome::Acquired(g) => g,
20677 crate::cache::AcquireOutcome::Busy(b) => {
20678 panic!("expected the direct acquire to win the lease first: {b:?}")
20679 }
20680 };
20681
20682 let mut state = RunState::new(
20683 repo.clone(),
20684 "main".to_owned(),
20685 head.clone(),
20686 "task".to_owned(),
20687 config,
20688 );
20689 state.candidates = vec![Candidate {
20690 index: 0,
20691 label: 'A',
20692 agent: "alpha".to_owned(),
20693 branch: "does-not-exist".to_owned(),
20694 worktree: repo.clone(),
20695 summary: String::new(),
20696 stat: String::new(),
20697 files: 0,
20698 commits: 0,
20699 empty: false,
20700 failed: None,
20701 verified_noop: None,
20702 duration_ms: 0,
20703 folded: false,
20704 }];
20705 state.tally = Some(Tally {
20706 first_choice: BTreeMap::from([('A', 1)]),
20707 borda: BTreeMap::new(),
20708 winner: 'A',
20709 rankings: 1,
20710 unanimous_initial: true,
20711 deliberated: false,
20712 changed_votes: 0,
20713 unanimous_final: true,
20714 tie_break: None,
20715 judges: 0,
20716 present: 0,
20717 quorum: 0,
20718 met_quorum: true,
20719 uncontested: Some("only candidate A produced a change".to_owned()),
20720 });
20721 state.reviews = vec![ReviewRound {
20725 round: 1,
20726 head: head.clone(),
20727 verified_head: None,
20728 verified_at: None,
20729 reviews: Vec::new(),
20730 e2e: Vec::new(),
20731 fix: None,
20732 blocking: 1,
20733 answered: 1,
20734 expected: 1,
20735 clean: false,
20736 verify_retried: false,
20737 e2e_deferred: true,
20738 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20739 progressed: false,
20740 vote_split: false,
20741 reconsideration: Vec::new(),
20742 verdict: None,
20743 }];
20744
20745 let mut runner = Runner {
20746 state,
20747 roles: ResolvedRoles {
20748 implementers: Vec::new(),
20749 judges: Vec::new(),
20750 reviewers: Vec::new(),
20751 fixer: None,
20752 conductor: conductor(),
20753 implementer_roster: Vec::new(),
20754 judge_roster: Vec::new(),
20755 reviewer_roster: Vec::new(),
20756 },
20757 sem: Arc::new(Semaphore::new(1)),
20758 pause: Pause::new(),
20759 interrupt: Pause::new(),
20760 };
20761
20762 let shell = runner.state.config.shell();
20763 runner
20764 .stop_reviewing("round budget spent", &shell, &repo)
20765 .await
20766 .expect("stop_reviewing");
20767
20768 let last = runner.state.reviews.last().expect("round record");
20769 assert_eq!(
20770 last.e2e_status(),
20771 E2eStatus::ResourceBlocked,
20772 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20773 failed: {last:?}"
20774 );
20775 assert_eq!(
20776 last.verified_head.as_deref(),
20777 Some(head.as_str()),
20778 "which commit this attempt targeted is known even though nothing finished checking \
20779 it"
20780 );
20781 let first_attempt_at = last
20782 .verified_at
20783 .expect("when this attempt ran is known too");
20784 assert_ne!(
20785 runner.state.status,
20786 RunStatus::Blocked,
20787 "contention is evidence about the machine, not the patch — it must not settle the \
20788 run as blocked: {:?}",
20789 runner.state.status
20790 );
20791 assert!(
20792 !runner
20793 .state
20794 .events
20795 .iter()
20796 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20797 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20798 runner.state.events
20799 );
20800
20801 runner
20806 .stop_reviewing("round budget spent", &shell, &repo)
20807 .await
20808 .expect("stop_reviewing retry");
20809 assert_eq!(
20810 runner.state.reviews.len(),
20811 1,
20812 "no new round was started: {:?}",
20813 runner.state.reviews
20814 );
20815 let last = runner.state.reviews.last().expect("round record");
20816 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20817 assert!(
20818 last.verified_at.expect("still known") > first_attempt_at,
20819 "a second reentry must be a fresh attempt, not a stale copy of the first"
20820 );
20821 assert_ne!(runner.state.status, RunStatus::Blocked);
20822
20823 held.release();
20824 }
20825
20826 #[cfg(unix)]
20827 #[tokio::test]
20828 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20829 ask_test_home();
20830 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20831 let tmp = tempfile::tempdir().unwrap();
20832 let repo = tmp.path().join("repo");
20833 std::fs::create_dir(&repo).unwrap();
20834 init_repo(&repo);
20835 let run_git = |args: &[&str]| {
20836 let out = std::process::Command::new("git")
20837 .args(args)
20838 .current_dir(&repo)
20839 .output()
20840 .unwrap();
20841 assert!(
20842 out.status.success(),
20843 "{args:?}: {}",
20844 String::from_utf8_lossy(&out.stderr)
20845 );
20846 };
20847 let origin = tmp.path().join("origin.git");
20848 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20849 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20850 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20851 run_git(&["add", "spec.rs"]);
20852 run_git(&["commit", "-m", "initial spec"]);
20853 run_git(&["push", "origin", "main"]);
20854 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20855 run_git(&["checkout", "-b", "fix/review"]);
20856 std::fs::write(
20857 repo.join("check.rs"),
20858 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20859 )
20860 .unwrap();
20861 run_git(&["add", "check.rs"]);
20862 run_git(&["commit", "-m", "use spec"]);
20863 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20864 run_git(&["checkout", "main"]);
20865 std::fs::write(
20866 repo.join("spec.rs"),
20867 "struct AgentSpec { max_turn_secs: u64 }\n",
20868 )
20869 .unwrap();
20870 run_git(&["add", "spec.rs"]);
20871 run_git(&["commit", "-m", "extend spec"]);
20872 run_git(&["push", "origin", "main"]);
20873 run_git(&["checkout", "fix/review"]);
20874
20875 let mut runner = runner_at(RunStatus::Reviewing);
20876 runner.state.repo = repo.clone();
20877 runner.state.base_commit = base;
20878 runner.state.pr_head = Some("fix/review".into());
20879 runner.state.config.graph.review_rounds = max_rounds;
20880 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20881 runner.state.config.graph.sessions = false;
20882 runner.state.config.disk.min_free_bytes = 0;
20883 runner.state.config.verify.e2e =
20884 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20885 runner.state.candidates = vec![Candidate {
20886 index: 0,
20887 label: 'A',
20888 agent: "fixture".into(),
20889 branch: "fix/review".into(),
20890 worktree: repo.clone(),
20891 summary: String::new(),
20892 stat: String::new(),
20893 files: 1,
20894 commits: 1,
20895 empty: false,
20896 failed: None,
20897 verified_noop: None,
20898 duration_ms: 0,
20899 folded: false,
20900 }];
20901 runner.state.tally = Some(Tally {
20902 first_choice: BTreeMap::new(),
20903 borda: BTreeMap::new(),
20904 winner: 'A',
20905 rankings: 0,
20906 unanimous_initial: true,
20907 deliberated: false,
20908 changed_votes: 0,
20909 unanimous_final: true,
20910 tie_break: None,
20911 judges: 0,
20912 present: 0,
20913 quorum: 0,
20914 met_quorum: true,
20915 uncontested: Some("review-only".into()),
20916 });
20917 let mut last = review_round(false, 0, 1, 1, false, false);
20918 last.head = reviewed.clone();
20919 last.e2e_deferred = true;
20920 last.e2e[0].resource_blocked = true;
20921 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20922 if max_rounds == 1 {
20923 last.e2e.clear();
20924 }
20925 runner.state.reviews = vec![last];
20926 let calls = tmp.path().join("fix-calls");
20927 let mut fixer = spec("repair");
20928 fixer
20929 .env
20930 .insert("FIX_CALLS".into(), calls.display().to_string());
20931 fixer.command = vec!["sh".into(), "-c".into(), r#"
20932 echo fix >> "$FIX_CALLS"
20933 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20934 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20935 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20936 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20937 "#.into()];
20938 let mut reviewer = spec("reviewer");
20939 reviewer.command = vec![
20940 "sh".into(),
20941 "-c".into(),
20942 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20943 .into(),
20944 ];
20945 runner.roles.fixer = Some(vec![fixer]);
20946 runner.roles.reviewers = vec![reviewer];
20947
20948 runner.sync_to_base().await.unwrap();
20949 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20950 assert_ne!(rebased, reviewed);
20951 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20952 if reentry {
20953 let shell = runner.state.config.shell();
20956 runner
20957 .stop_unresolved_review("deferred verification", &shell, &repo)
20958 .await
20959 .unwrap();
20960 assert!(runner.catchup_repairable());
20961 assert!(!calls.exists());
20962 runner.state = RunState::load(&runner.state.id).unwrap();
20963 assert!(runner.catchup_repairable());
20964 }
20965 runner.review_loop().await.unwrap();
20966 let first = &runner.state.reviews[0];
20967 assert_eq!(first.head, reviewed);
20968 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20969 assert!(
20970 first.verify_retried,
20971 "a reproduced build failure gets exactly one retry: {:?}",
20972 first.e2e
20973 );
20974 assert!(first.e2e[0].output_tail.contains("E0063"));
20975 assert!(!runner.catchup_repairable());
20976 if max_rounds == 2 {
20977 assert_eq!(
20978 runner.state.status,
20979 RunStatus::Gating,
20980 "{}",
20981 crate::report::run(&runner.state)
20982 );
20983 assert_eq!(runner.state.reviews.len(), 2);
20984 assert!(first.fix.as_ref().unwrap().committed);
20985 assert!(runner.state.reviews[1].clean);
20986 assert_eq!(
20987 runner
20988 .state
20989 .reviews
20990 .iter()
20991 .filter(|r| r.fix.is_some())
20992 .count(),
20993 1
20994 );
20995 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20996 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20997 } else {
20998 assert_eq!(runner.state.status, RunStatus::Blocked);
20999 assert!(first.fix.is_none());
21000 assert!(!calls.exists());
21001 let recovery = crate::recovery::assess(
21002 "prior",
21003 Some(&runner.state),
21004 &runner.landing_base(),
21005 crate::recovery::Relation::Diverged,
21006 );
21007 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21008 let reasons = recovery.reasons.join("\n");
21009 assert!(
21010 reasons.contains(&reviewed) && reasons.contains(&rebased),
21011 "{reasons}"
21012 );
21013 assert!(
21014 reasons.contains("base sync to")
21015 && reasons.contains("additional review is required"),
21016 "{reasons}"
21017 );
21018 }
21019 }
21020 }
21021
21022 #[tokio::test]
21034 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21035 crate::run::pin_test_home();
21036 let home = crate::run::home();
21037
21038 let tmp = tempfile::tempdir().expect("tempdir");
21039 let repo = tmp.path().join("repo");
21040 std::fs::create_dir_all(&repo).unwrap();
21041 init_repo(&repo);
21042 let head = crate::git::rev_parse(&repo, "HEAD")
21043 .await
21044 .expect("rev-parse");
21045 let cache_dir = tmp.path().join("target");
21046
21047 let mut config = Config::default();
21048 config.verify.e2e = vec![format!(
21049 "CARGO_TARGET_DIR='{}' test -f README.md",
21050 cache_dir.display()
21051 )];
21052 config.graph.review_rounds = 1;
21053 config.graph.timeout_verify = Some(2);
21054 config.graph.timeout_resource_wait = 2;
21055
21056 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21057 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21058 .expect("no io error acquiring directly")
21059 {
21060 crate::cache::AcquireOutcome::Acquired(g) => g,
21061 crate::cache::AcquireOutcome::Busy(b) => {
21062 panic!("expected the direct acquire to win the lease first: {b:?}")
21063 }
21064 };
21065
21066 let mut state = RunState::new(
21067 repo.clone(),
21068 "main".to_owned(),
21069 head.clone(),
21070 "task".to_owned(),
21071 config,
21072 );
21073 state.candidates = vec![Candidate {
21074 index: 0,
21075 label: 'A',
21076 agent: "alpha".to_owned(),
21077 branch: "does-not-exist".to_owned(),
21078 worktree: repo.clone(),
21079 summary: String::new(),
21080 stat: String::new(),
21081 files: 0,
21082 commits: 0,
21083 empty: false,
21084 failed: None,
21085 verified_noop: None,
21086 duration_ms: 0,
21087 folded: false,
21088 }];
21089 state.tally = Some(Tally {
21090 first_choice: BTreeMap::from([('A', 1)]),
21091 borda: BTreeMap::new(),
21092 winner: 'A',
21093 rankings: 1,
21094 unanimous_initial: true,
21095 deliberated: false,
21096 changed_votes: 0,
21097 unanimous_final: true,
21098 tie_break: None,
21099 judges: 0,
21100 present: 0,
21101 quorum: 0,
21102 met_quorum: true,
21103 uncontested: Some("only candidate A produced a change".to_owned()),
21104 });
21105 state.reviews = vec![ReviewRound {
21109 round: 1,
21110 head: head.clone(),
21111 verified_head: Some(head.clone()),
21112 verified_at: Some(jiff::Timestamp::now()),
21113 reviews: Vec::new(),
21114 e2e: vec![CommandOutcome {
21115 command: format!(
21116 "CARGO_TARGET_DIR='{}' test -f README.md",
21117 cache_dir.display()
21118 ),
21119 code: None,
21120 output_tail: "waiting for the shared build cache".to_owned(),
21121 duration_ms: 0,
21122 resource_blocked: true,
21123 }],
21124 fix: None,
21125 blocking: 1,
21126 answered: 1,
21127 expected: 1,
21128 clean: false,
21129 verify_retried: false,
21130 e2e_deferred: false,
21131 e2e_defer_reason: None,
21132 progressed: false,
21133 vote_split: false,
21134 reconsideration: Vec::new(),
21135 verdict: None,
21136 }];
21137
21138 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21139 let mut runner = Runner {
21140 state,
21141 roles: ResolvedRoles {
21142 implementers: Vec::new(),
21143 judges: Vec::new(),
21144 reviewers: Vec::new(),
21145 fixer: None,
21146 conductor: conductor(),
21147 implementer_roster: Vec::new(),
21148 judge_roster: Vec::new(),
21149 reviewer_roster: Vec::new(),
21150 },
21151 sem: Arc::new(Semaphore::new(1)),
21152 pause: Pause::new(),
21153 interrupt: Pause::new(),
21154 };
21155
21156 runner.review_loop().await.expect("review_loop");
21161
21162 assert_eq!(
21163 runner.state.reviews.len(),
21164 1,
21165 "no new round was started on top of the unresolved one: {:?}",
21166 runner.state.reviews
21167 );
21168 let last = &runner.state.reviews[0];
21169 assert_eq!(
21170 last.e2e_status(),
21171 E2eStatus::ResourceBlocked,
21172 "still contended: {last:?}"
21173 );
21174 assert!(
21175 last.verified_at.expect("still known") > first_attempt_at,
21176 "review_loop must have actually retried the check, not left it exactly as found"
21177 );
21178 assert_ne!(
21179 runner.state.status,
21180 RunStatus::Blocked,
21181 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21182 runner.state.status
21183 );
21184
21185 held.release();
21186 }
21187
21188 #[tokio::test]
21189 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21190 crate::run::pin_test_home();
21191 for status in [RunStatus::Blocked, RunStatus::Gating] {
21192 let mut runner = runner_at(status);
21193 runner.state.config.graph.land = true;
21194 runner.state.gate_ran = true;
21195 runner.state.merge = Some(MergeOutcome {
21196 mode: MergeMode::Pr,
21197 ok: false,
21198 detail: "the fixer produced no commit".into(),
21199 empty: false,
21200 });
21201 let url = "https://example.invalid/o/r/pull/1";
21202 runner.state.pr = Some(crate::run::PrRecord {
21203 url: url.into(),
21204 number: 1,
21205 state: "open".into(),
21206 checks: "red".into(),
21207 round: 1,
21208 rounds: 1,
21209 red_at_merge: Vec::new(),
21210 });
21211 runner.state.save().unwrap();
21212 runner.execute_graph().await.unwrap();
21216 assert_eq!(runner.state.status, RunStatus::Blocked);
21217 assert!(runner.state.land_sync_pending);
21218 assert!(runner.state.events.iter().any(|event|
21219 event.node == "land" && event.message == format!("watching {url}")));
21220 assert_eq!(
21221 runner.state.merge.as_ref().unwrap().detail,
21222 "the fixer produced no commit"
21223 );
21224 }
21225 }
21226
21227 #[tokio::test]
21228 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21229 crate::run::pin_test_home();
21230 let tmp = tempfile::tempdir().expect("tempdir");
21231 let repo = tmp.path().join("repo");
21232 std::fs::create_dir_all(&repo).unwrap();
21233 init_repo(&repo);
21234
21235 let mut config = Config::default();
21236 config.merge.mode = MergeMode::Pr;
21237 config.graph.land = true;
21238 config.graph.land_approval = false;
21239
21240 let mut state = RunState::new(
21241 repo.clone(),
21242 "main".to_owned(),
21243 "deadbeef".to_owned(),
21244 "task".to_owned(),
21245 config,
21246 );
21247 state.candidates = vec![Candidate {
21248 index: 0,
21249 label: 'A',
21250 agent: "alpha".to_owned(),
21251 branch: "does-not-exist".to_owned(),
21252 worktree: repo.clone(),
21253 summary: String::new(),
21254 stat: String::new(),
21255 files: 0,
21256 commits: 0,
21257 empty: false,
21258 failed: None,
21259 verified_noop: None,
21260 duration_ms: 0,
21261 folded: false,
21262 }];
21263 state.tally = Some(Tally {
21264 first_choice: BTreeMap::from([('A', 1)]),
21265 borda: BTreeMap::new(),
21266 winner: 'A',
21267 rankings: 1,
21268 unanimous_initial: true,
21269 deliberated: false,
21270 changed_votes: 0,
21271 unanimous_final: true,
21272 tie_break: None,
21273 judges: 0,
21274 present: 0,
21275 quorum: 0,
21276 met_quorum: true,
21277 uncontested: Some("only candidate A produced a change".to_owned()),
21278 });
21279 state.reviews = vec![ReviewRound {
21280 round: 1,
21281 head: "deadbeef".to_owned(),
21282 verified_head: None,
21283 verified_at: None,
21284 reviews: Vec::new(),
21285 e2e: Vec::new(),
21286 fix: None,
21287 blocking: 0,
21288 answered: 0,
21289 expected: 0,
21290 clean: true,
21291 verify_retried: false,
21292 e2e_deferred: false,
21293 e2e_defer_reason: None,
21294 progressed: false,
21295 vote_split: false,
21296 reconsideration: Vec::new(),
21297 verdict: None,
21298 }];
21299 state.gate = vec![CommandOutcome {
21300 command: "test".to_owned(),
21301 code: Some(0),
21302 output_tail: String::new(),
21303 duration_ms: 0,
21304 resource_blocked: false,
21305 }];
21306 state.gate_ran = true;
21307 state.status = RunStatus::Landing;
21311 state.merge = Some(MergeOutcome {
21312 mode: MergeMode::Pr,
21313 ok: true,
21314 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21315 empty: false,
21316 });
21317
21318 ask_test_home();
21322 state.save().expect("save landing run");
21325 let store = ask::Questions::open();
21326 let q = ask_open_question(&store, &state.id);
21327
21328 let mut runner = Runner {
21329 state,
21330 roles: ResolvedRoles {
21331 implementers: Vec::new(),
21332 judges: Vec::new(),
21333 reviewers: Vec::new(),
21334 fixer: None,
21335 conductor: conductor(),
21336 implementer_roster: Vec::new(),
21337 judge_roster: Vec::new(),
21338 reviewer_roster: Vec::new(),
21339 },
21340 sem: Arc::new(Semaphore::new(1)),
21341 pause: Pause::new(),
21342 interrupt: Pause::new(),
21343 };
21344
21345 runner.execute().await.expect("execute");
21350
21351 assert_eq!(
21352 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21353 Some("https://example.invalid/x/y/pull/1"),
21354 "reentry must not push again or open a second pull request over the \
21355 one `land` is already watching"
21356 );
21357 assert_ne!(
21358 runner.state.status,
21359 RunStatus::Landing,
21360 "land could not actually reach the fake pull request, so it must \
21361 have given up rather than left the run silently parked forever"
21362 );
21363 assert_eq!(runner.state.status, RunStatus::Blocked);
21367 assert!(
21368 store.get(&q.id).unwrap().status.open(),
21369 "Blocked is still alive; settle_questions must have been a no-op here"
21370 );
21371 }
21372
21373 fn state_with_round(round: ReviewRound) -> RunState {
21374 let mut s = RunState::new(
21375 PathBuf::from("/repo"),
21376 "main".to_owned(),
21377 "abc1234".to_owned(),
21378 "add retries".to_owned(),
21379 Config::default(),
21380 );
21381 s.reviews = vec![round];
21382 s
21383 }
21384
21385 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21386 crate::verdict::Finding {
21387 id: id.to_owned(),
21388 severity,
21389 file: None,
21390 line: None,
21391 title: title.to_owned(),
21392 detail: String::new(),
21393 }
21394 }
21395
21396 #[test]
21397 fn pr_body_names_open_findings_and_declined_ones() {
21398 let round = ReviewRound {
21399 round: 2,
21400 head: "deadbee".to_owned(),
21401 verified_head: None,
21402 verified_at: None,
21403 reviews: vec![ReviewRecord {
21404 attempts: 0,
21405 reviewer: 1,
21406 agent: "alpha".to_owned(),
21407 summary: String::new(),
21408 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21409 vote: None,
21410 failed: None,
21411 duration_ms: 0,
21412 }],
21413 e2e: vec![CommandOutcome {
21414 command: "cargo test".to_owned(),
21415 code: Some(0),
21416 output_tail: String::new(),
21417 duration_ms: 0,
21418 resource_blocked: false,
21419 }],
21420 verify_retried: false,
21421 e2e_deferred: false,
21422 e2e_defer_reason: None,
21423 fix: Some(FixRecord {
21424 agent: "alpha".to_owned(),
21425 addressed: Vec::new(),
21426 rejected: vec![crate::verdict::Rejection {
21427 id: "R1-1-1".to_owned(),
21428 why: "not reachable from any caller".to_owned(),
21429 }],
21430 notes: String::new(),
21431 committed: true,
21432 failed: None,
21433 duration_ms: 0,
21434 continuation: None,
21435 }),
21436 blocking: 0,
21437 answered: 1,
21438 expected: 1,
21439 clean: false,
21440 progressed: true,
21441 vote_split: false,
21442 reconsideration: Vec::new(),
21443 verdict: None,
21444 };
21445 let state = state_with_round(round);
21446 let body = pr_message(&state, 'A').body;
21447
21448 assert!(body.contains("add retries"), "the task must still be there");
21449 assert!(body.contains("R2-1-1"), "{body}");
21450 assert!(!body.contains("unused import"), "{body}");
21451 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21452 assert!(
21453 !body.contains("not reachable from any caller"),
21454 "the reason it was declined: {body}"
21455 );
21456 }
21457
21458 #[test]
21459 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21460 let round = ReviewRound {
21461 round: 1,
21462 head: "deadbee".to_owned(),
21463 verified_head: None,
21464 verified_at: None,
21465 reviews: vec![ReviewRecord {
21466 attempts: 0,
21467 reviewer: 1,
21468 agent: "alpha".to_owned(),
21469 summary: String::new(),
21470 findings: Vec::new(),
21471 vote: None,
21472 failed: None,
21473 duration_ms: 0,
21474 }],
21475 e2e: Vec::new(),
21476 verify_retried: false,
21477 e2e_deferred: false,
21478 e2e_defer_reason: None,
21479 fix: None,
21480 blocking: 0,
21481 answered: 1,
21482 expected: 1,
21483 clean: true,
21484 progressed: false,
21485 vote_split: false,
21486 reconsideration: Vec::new(),
21487 verdict: None,
21488 };
21489 let state = state_with_round(round);
21490 let body = pr_message(&state, 'A').body;
21491 assert!(!body.contains("Open review findings"), "{body}");
21492 assert!(!body.contains("Declined"), "{body}");
21493 }
21494
21495 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21496 let mut state = RunState::new(
21497 PathBuf::from("/repo"),
21498 "main".to_owned(),
21499 "abc1234".to_owned(),
21500 instruction.to_owned(),
21501 Config::default(),
21502 );
21503 state.config.disk.min_free_bytes = 0;
21505 state.candidates.push(Candidate {
21506 index: 0,
21507 label: 'A',
21508 agent: "alpha".to_owned(),
21509 branch: "magi/x/A".to_owned(),
21510 worktree: PathBuf::from("/wt"),
21511 summary: summary.to_owned(),
21512 stat: String::new(),
21513 files: 1,
21514 commits: 1,
21515 empty: false,
21516 failed: None,
21517 verified_noop: None,
21518 folded: false,
21519 duration_ms: 0,
21520 });
21521 state
21522 }
21523
21524 fn review_state(subjects: &[&str]) -> RunState {
21525 let mut state = state_with_summary(
21526 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21527 "",
21528 );
21529 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21530 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21531 state
21532 }
21533
21534 #[tokio::test]
21537 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21538 ask_test_home();
21539 let tmp = tempfile::tempdir().unwrap();
21540 let repo = tmp.path().join("repo");
21541 std::fs::create_dir_all(&repo).unwrap();
21542 init_repo(&repo);
21543 let origin = tmp.path().join("origin.git");
21544 let g = |dir: &Path, args: &[&str]| {
21545 let out = std::process::Command::new("git")
21546 .args(args)
21547 .current_dir(dir)
21548 .quiet()
21549 .output()
21550 .expect("spawn git");
21551 assert!(
21552 out.status.success(),
21553 "git {args:?}: {}",
21554 String::from_utf8_lossy(&out.stderr)
21555 );
21556 };
21557 g(
21558 tmp.path(),
21559 &[
21560 "clone",
21561 "--bare",
21562 "-q",
21563 repo.to_str().unwrap(),
21564 origin.to_str().unwrap(),
21565 ],
21566 );
21567 g(
21568 &repo,
21569 &["remote", "add", "origin", origin.to_str().unwrap()],
21570 );
21571 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21572
21573 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21575 g(&repo, &["add", "-A"]);
21576 g(
21577 &repo,
21578 &["commit", "-q", "-m", "chore(deps): update a crate"],
21579 );
21580 g(&repo, &["push", "-q", "origin", "main"]);
21581 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21582 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21583 g(&repo, &["add", "-A"]);
21584 g(
21585 &repo,
21586 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21587 );
21588 g(&repo, &["checkout", "-q", "main"]);
21589
21590 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21591 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21592 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21594 assert_eq!(fallback, c1);
21595
21596 let mut state = review_state(&[
21597 "chore(deps): update a crate",
21598 "fix(daemon): apply a chosen action",
21599 ]);
21600 state.repo = repo.clone();
21601 state.base_branch = "main".to_owned();
21602 state.base_commit = c1;
21603 refresh_reviewed_commits(&mut state, "feat/own").await;
21604 assert_eq!(
21605 state.reviewed_commits,
21606 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21607 );
21608 assert_eq!(
21609 review_title(&state).as_deref(),
21610 Some("fix(daemon): apply a chosen action")
21611 );
21612 assert_eq!(
21613 leaked_subjects(&state, "feat/own").await,
21614 Some(vec!["chore(deps): update a crate".to_owned()])
21615 );
21616 state.config.merge.remote = "nowhere".to_owned();
21619 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21620 }
21621
21622 struct SummaryFixture {
21626 _tmp: tempfile::TempDir,
21627 repo: PathBuf,
21628 reply: PathBuf,
21629 runner: Runner,
21630 }
21631
21632 fn summary_fixture() -> SummaryFixture {
21633 ask_test_home();
21634 let tmp = tempfile::tempdir().unwrap();
21635 let repo = tmp.path().join("repo");
21636 std::fs::create_dir_all(&repo).unwrap();
21637 init_repo(&repo);
21638 let origin = tmp.path().join("origin.git");
21639 let g = |args: &[&str]| {
21640 let out = std::process::Command::new("git")
21641 .args(args)
21642 .current_dir(&repo)
21643 .quiet()
21644 .output()
21645 .expect("spawn git");
21646 assert!(
21647 out.status.success(),
21648 "git {args:?}: {}",
21649 String::from_utf8_lossy(&out.stderr)
21650 );
21651 };
21652 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21653 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21654 g(&["push", "-q", "origin", "main"]);
21655 g(&["checkout", "-q", "-b", "feat/own"]);
21656 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21657 g(&["add", "-A"]);
21658 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21659 let reply = tmp.path().join("reply.txt");
21660 std::fs::write(
21661 &reply,
21662 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21663 )
21664 .unwrap();
21665 let mut runner = runner_at(RunStatus::Gating);
21666 let mut state = review_state(&["feat(own): add the own file"]);
21667 state.repo = repo.clone();
21668 state.base_branch = "main".to_owned();
21669 state.candidates[0].branch = "feat/own".to_owned();
21670 state.candidates[0].worktree = repo.clone();
21671 let mut author = spec("alpha");
21672 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21673 author
21674 .env
21675 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21676 state.config.agents = vec![author];
21677 runner.state = state;
21678 SummaryFixture {
21679 _tmp: tmp,
21680 repo,
21681 reply,
21682 runner,
21683 }
21684 }
21685
21686 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21687 let winner = f.runner.state.candidates[0].clone();
21688 f.runner
21689 .guarded_pr_message(&winner, None, true)
21690 .await
21691 .unwrap()
21692 .expect("an English description never parks")
21693 }
21694
21695 fn clean_round_with_minor_findings() -> ReviewRound {
21696 let mut round = passing_round(2, None);
21697 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21698 round.reviews = vec![ReviewRecord {
21699 attempts: 0,
21700 reviewer: 1,
21701 agent: "alpha".into(),
21702 summary: "内部レビュー".into(),
21703 findings: vec![
21704 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21705 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21706 ],
21707 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21708 failed: None,
21709 duration_ms: 0,
21710 }];
21711 round
21712 }
21713
21714 #[tokio::test]
21715 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21716 for review_only in [false, true] {
21717 let mut f = summary_fixture();
21718 if !review_only {
21719 f.runner.state.candidates[0].agent = "alpha".into();
21720 }
21721 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21722 assert!(f.runner.state.reviews[0].clean);
21723 assert!(f.runner.state.reviews[0].fix.is_none());
21724 assert!(f.runner.state.open_findings().is_empty());
21725 assert_eq!(
21726 crate::pr_outcome::required_finding_ids(&f.runner.state),
21727 ["R2-1-1", "R2-1-2"]
21728 );
21729 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21730 let evidence = crate::pr_outcome::summary_evidence(
21731 &f.runner.state,
21732 crate::pr_outcome::FinalHead::At(&head),
21733 );
21734 assert!(
21735 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21736 );
21737 let omitted = posted(&mut f).await;
21739 assert!(omitted.body.contains("No description"));
21740 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21741 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21742 assert!(f.runner.state.change_summary.is_none());
21743 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21744 let described = posted(&mut f).await;
21745 assert!(described.body.contains("improve documentation for callers"));
21746 assert!(described.body.contains("remaining edge case"));
21747 assert!(described.body.contains("## Open review findings"));
21748 assert!(!described.body.contains("未修正"));
21749 }
21750 }
21751
21752 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21753 let mut f = summary_fixture();
21754 if !review_only {
21755 f.runner.state.candidates[0].agent = "alpha".into();
21756 }
21757 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21758 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21759 fixed
21760 .fix
21761 .as_mut()
21762 .unwrap()
21763 .rejected
21764 .push(crate::verdict::Rejection {
21765 id: "R1-1-1".into(),
21766 why: "呼び出されないコードのため辞退".into(),
21767 });
21768 let mut clean = clean_round_with_minor_findings();
21769 clean.verified_head = Some(head.clone());
21770 f.runner.state.reviews = vec![fixed, clean];
21771 ok_gate(&mut f.runner.state, Some(&head));
21772 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21773 f.runner.state.config.agents.push(scripted("beta", reply));
21774 f.runner.state.config.roles.language_judge =
21775 Some(crate::config::AgentChoice::One("beta".into()));
21776 f
21777 }
21778
21779 #[tokio::test]
21780 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21781 for review_only in [false, true] {
21782 let mut f = rewrite_fixture_with_findings(review_only,
21783 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21784 let m = posted(&mut f).await;
21785 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21786 let expected =
21787 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21788 assert!(m.body.contains(&expected));
21789 assert!(m.body.contains("## Review rounds"));
21790 assert!(
21791 m.body
21792 .contains("Gate (`verify.gate`): passed on the final commit")
21793 );
21794 assert!(
21795 m.body
21796 .contains("Tests (`verify.e2e`): passed on the final commit")
21797 );
21798 let c = f.runner.state.change_summary.as_ref().unwrap();
21799 assert!(
21800 m.body
21801 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21802 );
21803 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21804 assert!(crate::pr_outcome::covers_required_findings(
21805 &f.runner.state,
21806 &m.body
21807 ));
21808 assert!(m.body.contains("documentation remains incomplete"));
21809 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21810 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21811 assert!(f.runner.state.github_text.is_none());
21812 }
21813 }
21814
21815 #[tokio::test]
21816 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21817 for review_only in [false, true] {
21818 for reply in [
21819 "TITLE: fix: public description\nA short English description.",
21820 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21821 ] {
21822 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21823 let winner = f.runner.state.candidates[0].clone();
21824 let posted = f
21825 .runner
21826 .guarded_pr_message(&winner, None, true)
21827 .await
21828 .unwrap();
21829 assert!(posted.is_none());
21830 assert!(f.runner.state.parked);
21831 let withheld = f.runner.state.github_text.as_ref().unwrap();
21832 assert!(
21833 withheld.rewrite.is_none(),
21834 "an incomplete rewrite must not be selectable"
21835 );
21836 let q = ask::Questions::open()
21837 .get(withheld.question.as_ref().unwrap())
21838 .unwrap();
21839 assert!(
21840 !q.choices
21841 .contains(&crate::github_text::USE_REWRITE.to_owned())
21842 );
21843 assert!(
21844 q.detail.contains("R1-1-1")
21845 && q.detail.contains("R2-1-1")
21846 && q.detail.contains("R2-1-2")
21847 );
21848 }
21849 }
21850 }
21851
21852 #[tokio::test]
21853 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21854 for review_only in [false, true] {
21855 let mut f = summary_fixture();
21856 if !review_only {
21857 f.runner.state.candidates[0].agent = "alpha".into();
21858 }
21859 f.runner.state.instruction = "内部タスクの説明".into();
21860 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21861 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21862 f.runner.state.reviews = vec![
21863 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21864 round_with_notes(2, Some("二回目の内部修正報告です。")),
21865 passing_round(3, Some(&head)),
21866 ];
21867 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21868 ok_gate(&mut f.runner.state, Some(&head));
21869 let captured = f.repo.join("public-prompt.txt");
21870 let author = &mut f.runner.state.config.agents[0];
21871 author.command = vec![
21872 "sh".into(),
21873 "-c".into(),
21874 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21875 ];
21876 author.env.insert(
21877 "PROMPT_FILE".into(),
21878 captured.to_string_lossy().into_owned(),
21879 );
21880 let m = posted(&mut f).await;
21881 assert!(m.body.contains("Adds `own.txt`"));
21882 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21883 assert!(m.body.contains("R1-1-1"));
21884 assert!(
21885 m.body
21886 .contains("Gate (`verify.gate`): passed on the final commit")
21887 );
21888 assert!(
21889 m.body
21890 .contains("Tests (`verify.e2e`): passed on the final commit")
21891 );
21892 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21893 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21894 let prompt = std::fs::read_to_string(captured).unwrap();
21895 assert!(prompt.contains(&format!(
21896 "git diff {} {head}",
21897 f.runner.state.change_summary.as_ref().unwrap().base
21898 )));
21899 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21900 assert_eq!(
21901 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21902 "二回目の内部修正報告です。"
21903 );
21904 }
21905 }
21906
21907 #[tokio::test]
21908 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21909 for review_only in [false, true] {
21910 let mut f = summary_fixture();
21911 if !review_only {
21912 f.runner.state.candidates[0].agent = "alpha".into();
21913 }
21914 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21915 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21916 std::fs::write(&f.reply, "no usable description").unwrap();
21917 let failed = posted(&mut f).await;
21918 assert!(
21919 failed
21920 .body
21921 .contains("No description of what this change does could be generated")
21922 );
21923 assert!(failed.body.is_ascii());
21924 assert!(f.runner.state.github_text.is_none());
21925 assert!(f.runner.state.change_summary.is_none());
21926 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21927 let evidence = crate::pr_outcome::summary_evidence(
21928 &f.runner.state,
21929 crate::pr_outcome::FinalHead::At(&head),
21930 );
21931 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21932 assert!(
21933 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21934 .exists()
21935 );
21936 f.runner.state =
21937 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21938 std::fs::write(
21939 &f.reply,
21940 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21941 )
21942 .unwrap();
21943 let retried = posted(&mut f).await;
21944 assert!(retried.body.contains("Adds the committed own file."));
21945 assert_eq!(
21946 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21947 "日本語の修正報告"
21948 );
21949 }
21950 }
21951
21952 #[tokio::test]
21953 async fn same_head_verification_and_findings_invalidate_public_prose() {
21954 let mut f = summary_fixture();
21955 posted(&mut f).await;
21956 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21957 let original = f
21958 .runner
21959 .state
21960 .change_summary
21961 .as_ref()
21962 .unwrap()
21963 .evidence
21964 .clone();
21965 ok_gate(&mut f.runner.state, Some(&head));
21966 std::fs::write(
21967 &f.reply,
21968 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21969 )
21970 .unwrap();
21971 let verified = posted(&mut f).await;
21972 assert!(verified.body.contains("now verified"));
21973 assert_ne!(
21974 f.runner.state.change_summary.as_ref().unwrap().evidence,
21975 original
21976 );
21977 let mut round = round_with_notes(1, Some("内部報告"));
21978 round.reviews.push(ReviewRecord {
21979 attempts: 0,
21980 reviewer: 1,
21981 agent: "alpha".into(),
21982 summary: "内部レビュー".into(),
21983 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21984 vote: None,
21985 failed: None,
21986 duration_ms: 0,
21987 });
21988 round
21989 .fix
21990 .as_mut()
21991 .unwrap()
21992 .rejected
21993 .push(crate::verdict::Rejection {
21994 id: "R1-1-1".into(),
21995 why: "呼び出し元から到達できない".into(),
21996 });
21997 f.runner.state.reviews = vec![round];
21998 let omitted = posted(&mut f).await;
22000 assert!(omitted.body.contains("No description"));
22001 assert!(omitted.body.contains("R1-1-1"));
22002 assert!(!omitted.body.contains("到達"));
22003 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22004 let described = posted(&mut f).await;
22005 assert!(described.body.contains("unreachable from callers"));
22006 assert!(described.body.contains("R1-1-1"));
22007 assert!(!described.body.contains("到達"));
22008 let evidence = f
22009 .runner
22010 .state
22011 .change_summary
22012 .as_ref()
22013 .unwrap()
22014 .evidence
22015 .clone();
22016 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22017 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22018 assert!(posted(&mut f).await.body.contains("rationale updated"));
22019 assert_ne!(
22020 f.runner.state.change_summary.as_ref().unwrap().evidence,
22021 evidence
22022 );
22023 f.runner
22025 .state
22026 .change_summary
22027 .as_mut()
22028 .unwrap()
22029 .evidence
22030 .clear();
22031 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22032 assert!(posted(&mut f).await.body.contains("current specification"));
22033 }
22034
22035 #[tokio::test]
22036 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22037 let mut f = summary_fixture();
22038 let m = posted(&mut f).await;
22039 assert_eq!(m.title, "feat(own): add the own file");
22040 assert!(
22041 m.body
22042 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22043 "{}",
22044 m.body
22045 );
22046 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22047 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22048 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22049 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22050 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22051 let kept = f.runner.state.change_summary.as_ref().unwrap();
22052 assert_eq!(
22053 (kept.head.as_str(), kept.base.as_str()),
22054 (head.as_str(), base.as_str())
22055 );
22056 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22057 }
22058
22059 #[tokio::test]
22060 async fn review_only_summary_receives_the_configured_github_language() {
22061 let mut f = summary_fixture();
22062 f.runner.state.config.graph.github_language = "ja".into();
22063 let captured = f.repo.join("summary-prompt.txt");
22064 let author = &mut f.runner.state.config.agents[0];
22065 author.command = vec![
22066 "sh".into(),
22067 "-c".into(),
22068 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22069 ];
22070 author.env.insert(
22071 "PROMPT_FILE".into(),
22072 captured.to_string_lossy().into_owned(),
22073 );
22074 std::fs::write(
22075 &f.reply,
22076 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22077 )
22078 .unwrap();
22079 let m = posted(&mut f).await;
22080 let prompt = std::fs::read_to_string(captured).unwrap();
22081 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22082 assert!(!prompt.contains("in English"));
22083 assert_eq!(m.title, "feat(own): ファイルを追加");
22084 assert!(m.body.contains("own.txt を追加します。"));
22085 }
22086
22087 #[tokio::test]
22088 async fn review_fixes_come_after_the_whole_change_summary() {
22089 let mut f = summary_fixture();
22090 f.runner.state.reviews = vec![
22091 round_with_notes(1, Some("handled the PATH case")),
22092 round_with_notes(2, None),
22093 ];
22094 let m = posted(&mut f).await;
22095 let summary = m.body.find("## Summary").expect("summary");
22096 let fixes = m.body.find("## Review rounds").expect("review rounds");
22097 assert!(summary < fixes, "{}", m.body);
22098 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22099 assert!(m.body.contains("## Final verification"), "{}", m.body);
22100 }
22101
22102 #[tokio::test]
22103 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22104 let mut f = summary_fixture();
22105 let first = posted(&mut f).await;
22106 let first_head = f.runner.state.change_summary.clone().unwrap().head;
22107 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22109 let again = posted(&mut f).await;
22110 assert_eq!((again.title, again.body), (first.title, first.body));
22111
22112 let g = |args: &[&str]| {
22113 let out = std::process::Command::new("git")
22114 .args(args)
22115 .current_dir(&f.repo)
22116 .quiet()
22117 .output()
22118 .expect("spawn git");
22119 assert!(out.status.success(), "git {args:?}");
22120 };
22121 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22122 g(&["add", "-A"]);
22123 g(&["commit", "-q", "-m", "fix(own): handle more"]);
22124 let second = posted(&mut f).await;
22125 let kept = f.runner.state.change_summary.clone().unwrap();
22126 assert_ne!(kept.head, first_head);
22127 assert!(second.body.contains("Not read yet."), "{}", second.body);
22128 assert!(!second.body.contains("so the first summary is read"));
22129 assert_eq!(second.title, "feat(own): other");
22130 }
22131
22132 #[tokio::test]
22133 async fn an_unavailable_summary_is_stated_and_never_invented() {
22134 let mut f = summary_fixture();
22135 std::fs::write(&f.reply, "no title line here").unwrap();
22136 let m = posted(&mut f).await;
22137 assert!(
22138 m.body
22139 .contains("No description of what this change does could be generated"),
22140 "{}",
22141 m.body
22142 );
22143 assert!(m.body.contains("internal reports remain in local records"));
22144 assert!(f.runner.state.change_summary.is_none());
22145 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22146 let winner = f.runner.state.candidates[0].clone();
22148 let preview = f
22149 .runner
22150 .guarded_pr_message(&winner, None, false)
22151 .await
22152 .unwrap()
22153 .unwrap();
22154 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22155 }
22156
22157 #[test]
22158 fn pr_message_review_single_commit_uses_its_subject() {
22159 let state = review_state(&["feat(nats): per-role user"]);
22160 let m = pr_message(&state, 'A');
22161 assert_eq!(m.title, "feat(nats): per-role user");
22162 assert!(!m.body.contains("Review the work already"), "{}", m.body);
22163 assert!(m.body.contains("## Commits under review"), "{}", m.body);
22164 }
22165
22166 #[test]
22167 fn pr_message_review_multi_commit_takes_the_oldest() {
22168 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22169 let m = pr_message(&state, 'A');
22170 assert_eq!(m.title, "feat: the change");
22171 for s in ["feat: the change", "fix: typo", "fix: again"] {
22172 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22173 }
22174 }
22175
22176 #[test]
22177 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22178 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22179 let mut state = review_state(&[first, "fix: later fixup"]);
22180 state.candidates[0].branch = "機能/ログイン".to_owned();
22181 let m = pr_message(&state, 'A');
22182 assert!(
22183 m.title.starts_with("chore: land candidate A of run"),
22184 "{}",
22185 m.title
22186 );
22187 }
22188 }
22189
22190 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22191 BranchFacts {
22192 commits: commits
22193 .iter()
22194 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22195 .collect(),
22196 stat: stat.to_owned(),
22197 }
22198 }
22199
22200 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22201 let mut r = review_round(true, 0, 1, 1, true, true);
22202 r.round = round;
22203 r.fix = notes.map(|n| FixRecord {
22204 agent: "fixer".to_owned(),
22205 addressed: Vec::new(),
22206 rejected: Vec::new(),
22207 notes: n.to_owned(),
22208 committed: true,
22209 failed: None,
22210 duration_ms: 0,
22211 continuation: None,
22212 });
22213 r
22214 }
22215
22216 #[test]
22217 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22218 let state = review_state(&["ignored"]);
22219 let f = facts(
22220 &[
22221 (
22222 "fix(login): resolve PATH on macOS",
22223 "Login shells skip rc files.",
22224 ),
22225 ("fix: address review", ""),
22226 ],
22227 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22228 );
22229 let m = pr_message_with(&state, 'A', Some(&f));
22230 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22231 assert!(
22232 m.body
22233 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22234 "{}",
22235 m.body
22236 );
22237 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22238 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22239 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22240 for banned in [
22241 "Review the work already",
22242 "no task statement",
22243 "Original task",
22244 ] {
22245 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22246 }
22247 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22248 }
22249
22250 #[test]
22251 fn pr_message_review_truncates_a_huge_first_commit_body() {
22252 let state = review_state(&["ignored"]);
22253 let f = facts(
22254 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22255 "s",
22256 );
22257 let m = pr_message_with(&state, 'A', Some(&f));
22258 assert!(m.body.len() < 30_000, "{}", m.body.len());
22259 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22260 assert!(
22261 m.body.contains("- ... 1 more commit(s)"),
22262 "{}",
22263 m.body.len()
22264 );
22265 assert!(m.body.ends_with("magi:candidate-a\n"));
22266 }
22267
22268 #[test]
22269 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22270 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22271 assert_eq!(m.title, "feat: x");
22272 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22273 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22274 assert!(!m.body.contains("no task statement"), "{}", m.body);
22275 }
22276
22277 #[test]
22278 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22279 let mut state = review_state(&["日本語の件名"]);
22280 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22281 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22282 state.candidates[0].branch = "機能/ログイン".to_owned();
22283 assert!(
22284 pr_message(&state, 'A')
22285 .title
22286 .starts_with("chore: land candidate A")
22287 );
22288 }
22289
22290 #[test]
22291 fn pr_message_review_fixes_survive_a_clean_final_round() {
22292 let mut state = review_state(&["feat: x"]);
22293 state.reviews = vec![
22294 round_with_notes(1, Some("handled the PATH case")),
22295 round_with_notes(2, None),
22296 ];
22297 let body = pr_message(&state, 'A').body;
22298 assert!(
22299 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22300 "{body}"
22301 );
22302 assert!(body.contains("### Round 1"), "{body}");
22303
22304 state.reviews = vec![
22305 round_with_notes(1, Some("first fix")),
22306 round_with_notes(2, Some("")),
22307 round_with_notes(3, Some("second fix")),
22308 round_with_notes(4, None),
22309 ];
22310 let body = pr_message(&state, 'A').body;
22311 assert!(
22312 body.contains("### Round 1") && !body.contains("first fix"),
22313 "{body}"
22314 );
22315 assert!(
22316 body.contains("### Round 3") && !body.contains("second fix"),
22317 "{body}"
22318 );
22319 assert!(body.contains("### Round 2"), "{body}");
22320 }
22321
22322 const FINAL: &str = "abcdef1234567890";
22323
22324 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22325 let mut r = round_with_notes(round, None);
22326 r.verified_head = verified.map(str::to_owned);
22327 r
22328 }
22329
22330 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22331 state.gate = vec![CommandOutcome {
22332 command: "gate".to_owned(),
22333 code: Some(0),
22334 output_tail: String::new(),
22335 duration_ms: 0,
22336 resource_blocked: false,
22337 }];
22338 state.gate_ran = true;
22339 state.gate_head = head.map(str::to_owned);
22340 }
22341
22342 fn body_at(state: &RunState) -> String {
22343 pr_message_raw(
22344 state,
22345 'A',
22346 None,
22347 crate::pr_outcome::FinalHead::At(FINAL),
22348 ChangeNote::NotAsked,
22349 )
22350 .body
22351 }
22352
22353 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22354
22355 #[test]
22356 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22357 let mut state = review_state(&["feat: x"]);
22358 state.reviews = vec![
22359 round_with_notes(1, Some(SANDBOXED)),
22360 passing_round(2, Some(FINAL)),
22361 ];
22362 ok_gate(&mut state, Some(FINAL));
22363 let body = body_at(&state);
22364 assert!(!body.contains("Handled the PATH case."), "{body}");
22365 assert!(!body.contains("could not commit"), "{body}");
22366 assert!(!body.contains("could not run"), "{body}");
22367 assert!(
22368 body.contains("The fix is committed on the branch"),
22369 "{body}"
22370 );
22371 assert!(!body.contains("sandbox"), "{body}");
22372 assert!(
22373 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22374 "{body}"
22375 );
22376 assert!(
22377 body.contains("Gate (`verify.gate`): passed on the final commit"),
22378 "{body}"
22379 );
22380 assert!(
22382 state.reviews[0]
22383 .fix
22384 .as_ref()
22385 .unwrap()
22386 .notes
22387 .contains("could not commit")
22388 );
22389 }
22390
22391 #[test]
22392 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22393 let mut state = review_state(&["feat: x"]);
22394 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22395 r1.fix.as_mut().unwrap().committed = false;
22396 let mut r2 = passing_round(2, Some(FINAL));
22397 r2.e2e[0].code = Some(1);
22398 state.reviews = vec![r1, r2];
22399 let body = body_at(&state);
22400 assert!(!body.contains("could not commit"), "{body}");
22401 assert!(!body.contains("could not run"), "{body}");
22402 assert!(body.contains("This round produced no commit"), "{body}");
22403 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22404 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22405 assert!(!body.contains("passed"), "{body}");
22406 }
22407
22408 #[test]
22409 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22410 let mut state = review_state(&["feat: x"]);
22411 state.reviews = vec![
22412 round_with_notes(1, Some(SANDBOXED)),
22413 passing_round(2, Some("0123456789abcdef")),
22414 ];
22415 ok_gate(&mut state, Some("0123456789abcdef"));
22416 let body = body_at(&state);
22417 assert!(!body.contains("HTTP tests could not run"), "{body}");
22418 assert!(!body.contains("could not commit"), "{body}");
22420 assert!(
22421 body.contains("passed on an earlier commit (`0123456`)"),
22422 "{body}"
22423 );
22424 assert!(!body.contains("passed on the final commit"), "{body}");
22425
22426 ok_gate(&mut state, None);
22428 let body = body_at(&state);
22429 assert!(
22430 body.contains("passed on a commit that was not recorded"),
22431 "{body}"
22432 );
22433 }
22434
22435 #[test]
22436 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22437 let state = state_with_summary(
22438 "add retries to the client",
22439 "TITLE: feat: retries\n\nDid it.",
22440 );
22441 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22442 assert_eq!(m.title, "feat: retries");
22443 assert!(
22444 m.body.contains("<summary>Original task</summary>"),
22445 "{}",
22446 m.body
22447 );
22448 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22449 assert!(
22450 m.body
22451 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22452 "{}",
22453 m.body
22454 );
22455 }
22456
22457 #[test]
22458 fn pr_message_review_bounds_a_long_english_subject() {
22459 let long = format!("feat: {}", "word ".repeat(100));
22460 let m = pr_message(&review_state(&[&long]), 'A');
22461 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22462 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22463 }
22464
22465 #[test]
22466 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22467 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22468 let m = pr_message(&state, 'A');
22469 assert_eq!(m.title, "add retries");
22470 assert!(m.body.contains("<summary>Original task</summary>"));
22471 assert!(!m.body.contains("Commits under review"));
22472 assert_eq!(landing_subject_source(&state), state.instruction);
22473 }
22474
22475 #[test]
22476 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22477 let state = review_state(&["feat: the change", "fix: typo"]);
22478 let source = landing_subject_source(&state);
22479 assert_eq!(land::merge_subject("", &source), "feat: the change");
22480 assert_eq!(
22481 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22482 "feat: the change"
22483 );
22484 assert_eq!(
22486 land::merge_subject("feat: renamed by hand", &source),
22487 "feat: renamed by hand"
22488 );
22489 let blank = review_state(&["日本語"]);
22490 assert!(
22491 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22492 );
22493 }
22494
22495 #[test]
22496 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22497 let state = review_state(&["feat: the change"]);
22498 let source = landing_subject_source(&state);
22499 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22500 assert_eq!(
22501 land::merge_subject(landing_title(&state, old), &source),
22502 "feat: the change"
22503 );
22504 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22505 let task = state_with_summary("add retries", "");
22506 assert_eq!(landing_title(&task, old), old);
22507 }
22508
22509 #[test]
22510 fn pr_message_describes_the_change_not_the_task() {
22511 let state = state_with_summary(
22512 "今回やってほしいこと: results projector を直す",
22513 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22514 );
22515 let m = pr_message(&state, 'A');
22516 assert_eq!(m.title, "fix(web): batch the runs list reads");
22517 assert!(
22518 m.body.starts_with("## Summary\n\n- reads run.json once"),
22519 "{}",
22520 m.body
22521 );
22522 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22523 let task_at = m.body.find("今回やってほしいこと").unwrap();
22524 let details_at = m.body.find("<details>").unwrap();
22525 assert!(
22526 details_at < task_at,
22527 "the task lives inside <details>: {}",
22528 m.body
22529 );
22530 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22531 assert!(m.body.contains("magi:candidate-a"));
22532 }
22533
22534 #[test]
22535 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22536 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22537 let m = pr_message(&state, 'A');
22538 assert_eq!(m.title, "add retries");
22539 assert!(
22540 m.body.contains("## Summary\n\n- did some things"),
22541 "{}",
22542 m.body
22543 );
22544
22545 let none = RunState::new(
22546 PathBuf::from("/repo"),
22547 "main".to_owned(),
22548 "abc1234".to_owned(),
22549 "add retries".to_owned(),
22550 Config::default(),
22551 );
22552 let m = pr_message(&none, 'A');
22553 assert_eq!(m.title, "add retries");
22554 assert!(!m.body.contains("## Summary"), "{}", m.body);
22555 }
22556
22557 #[test]
22558 fn pr_message_refuses_the_candidate_commit_subject() {
22559 for bad in [
22560 "TITLE: magi: candidate A (uncommitted work)",
22561 "TITLE: chore: stuff (uncommitted work)",
22562 "TITLE: ",
22563 ] {
22564 let state = state_with_summary("add retries", bad);
22565 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22566 }
22567 }
22568
22569 #[test]
22570 fn pr_message_bounds_a_very_long_task_and_title() {
22571 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22572 let state = state_with_summary(&long, "- nothing");
22573 let m = pr_message(&state, 'A');
22574 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22575 assert!(!m.title.contains('\n'));
22576
22577 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22578 let m = pr_message(&state, 'A');
22579 assert!(m.title.starts_with("feat: "));
22580 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22581 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22582 }
22583
22584 fn long_title_of(instruction: &str) -> String {
22585 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22586 }
22587
22588 #[test]
22589 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22590 let first = "Make the landing path keep a readable title for long tasks";
22591 let line = format!(
22592 "{first}. {}",
22593 "And then keep going with more words ".repeat(20)
22594 );
22595 let t = long_title_of(&line);
22596 assert_eq!(t, first);
22597 assert!(!t.starts_with("chore: land"));
22598 }
22599
22600 #[test]
22601 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22602 let line = "word ".repeat(200);
22603 let t = long_title_of(&line);
22604 assert!(t.ends_with("word..."), "{t}");
22605 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22606 }
22607
22608 #[test]
22609 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22610 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22611 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22612 }
22613 }
22614
22615 #[test]
22616 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22617 let mut state = state_with_summary("再試行を追加する", "- nothing");
22618 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22619 state.config.graph.github_language = "ja".into();
22620 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22621 }
22622
22623 #[test]
22624 fn pr_message_title_limit_is_exact() {
22625 let at = "a".repeat(PR_TITLE_MAX);
22626 assert_eq!(long_title_of(&at), at);
22627 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22628 assert!(over.ends_with("..."), "{over}");
22629 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22630 }
22631
22632 #[test]
22633 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22634 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22635 let t = long_title_of(&line);
22636 assert!(t.ends_with("..."), "{t}");
22637 assert!(t.is_ascii(), "{t}");
22638 }
22639
22640 #[test]
22641 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22642 let line = format!(
22643 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22644 "plus more filler words ".repeat(20)
22645 );
22646 let t = long_title_of(&line);
22647 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22648 }
22649
22650 #[test]
22651 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22652 let line = format!("\n\n# {}", "title words ".repeat(40));
22653 let state = state_with_summary(&line, "- nothing");
22654 let m = pr_message(&state, 'A');
22655 assert!(m.title.starts_with("title words"), "{}", m.title);
22656 assert_eq!(
22657 land::merge_subject(&m.title, &landing_subject_source(&state)),
22658 m.title
22659 );
22660 assert_eq!(
22662 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22663 "feat: renamed by hand"
22664 );
22665 }
22666
22667 #[tokio::test]
22668 async fn github_text_rewrite_is_bounded_and_falls_back() {
22669 crate::run::pin_test_home();
22670 for (reply, accepted) in [
22671 (
22672 "TITLE: fix: retries\nAdd retries for failed requests.",
22673 true,
22674 ),
22675 (
22676 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22677 false,
22678 ),
22679 (
22680 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22681 true,
22682 ),
22683 (
22684 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22685 true,
22686 ),
22687 (
22688 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22689 true,
22690 ),
22691 (
22692 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22693 false,
22694 ),
22695 ("TITLE: fix: retries\n", false),
22696 ] {
22697 let dir = tempfile::tempdir().unwrap();
22698 let mut runner = runner_at(RunStatus::Gating);
22699 runner.state = state_with_summary(
22700 "add retries",
22701 "TITLE: fix: retries\n日本語の説明を書きます。",
22702 );
22703 runner.state.repo = dir.path().to_owned();
22704 runner.state.candidates[0].worktree = dir.path().to_owned();
22705 let mut author = spec("alpha");
22706 author.command = vec![
22707 "sh".into(),
22708 "-c".into(),
22709 "printf '%s' \"$REWRITE_REPLY\"".into(),
22710 ];
22711 author.env.insert("REWRITE_REPLY".into(), reply.into());
22712 runner.state.config.agents = vec![author];
22713 seed_public_summary(&mut runner);
22714 let winner = runner.state.candidates[0].clone();
22715 let first = runner
22716 .guarded_pr_message(&winner, None, true)
22717 .await
22718 .unwrap();
22719 let message = if accepted {
22720 first.expect("a passing rewrite never asks")
22721 } else {
22722 assert!(first.is_none());
22724 assert!(runner.state.parked);
22725 let store = ask::Questions::open();
22726 let asked: Vec<_> = store
22727 .list()
22728 .into_iter()
22729 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22730 .collect();
22731 assert_eq!(asked.len(), 1);
22732 assert!(!asked[0].detail.contains("secret"));
22733 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22734 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22735 assert_eq!(
22736 asked[0]
22737 .choices
22738 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22739 rewrite.is_some()
22740 );
22741 store
22744 .update(&asked[0].id, |q| {
22745 q.abandon("test");
22746 Ok(())
22747 })
22748 .unwrap();
22749 let again = runner
22750 .guarded_pr_message(&winner, None, true)
22751 .await
22752 .unwrap()
22753 .expect("an abandoned question falls back");
22754 let n = store
22755 .list()
22756 .into_iter()
22757 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22758 .count();
22759 assert_eq!(n, 1, "asked once per rejected text");
22760 again
22761 };
22762 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22763 assert!(!message.body.contains("secret"));
22764 assert!(!message.body.contains("prefix"));
22765 assert!(!message.title.contains("secret"));
22766 assert_eq!(
22767 message.body.contains("Add retries for failed requests."),
22768 accepted
22769 );
22770 assert_eq!(
22771 runner.state.seats["github-text-rewrite"].turns, 1,
22772 "only one rewrite invocation"
22773 );
22774 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22775 if accepted {
22776 assert!(runner.state.github_text.is_none());
22777 assert!(!runner.state.parked);
22778 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22779 assert!(message.body.contains("## Final verification"));
22780 assert!(message.body.contains("Evidence: final diff"));
22781 assert!(
22782 message
22783 .body
22784 .contains(&format!("magi:run/{}", runner.state.id))
22785 );
22786 }
22787 if !accepted {
22788 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22789 assert!(
22790 message
22791 .body
22792 .contains(&format!("magi:run/{}", runner.state.id))
22793 );
22794 }
22795 }
22796 }
22797
22798 fn scripted(id: &str, reply: &str) -> AgentSpec {
22799 let mut a = spec(id);
22800 a.command = vec![
22801 "sh".into(),
22802 "-c".into(),
22803 "printf '%s' \"$REWRITE_REPLY\"".into(),
22804 ];
22805 a.env.insert("REWRITE_REPLY".into(), reply.into());
22806 a
22807 }
22808
22809 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22810 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22811
22812 fn seed_public_summary(runner: &mut Runner) {
22814 let repo = &runner.state.repo;
22815 let g = |args: &[&str]| {
22816 let out = std::process::Command::new("git")
22817 .args(args)
22818 .current_dir(repo)
22819 .quiet()
22820 .output()
22821 .unwrap();
22822 assert!(out.status.success(), "git {args:?}");
22823 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22824 };
22825 g(&["init", "-q", "-b", "main"]);
22826 g(&[
22827 "-c",
22828 "user.name=Test",
22829 "-c",
22830 "user.email=test@example.invalid",
22831 "commit",
22832 "-q",
22833 "--allow-empty",
22834 "-m",
22835 "feat: fixture",
22836 ]);
22837 let head = g(&["rev-parse", "HEAD"]);
22838 g(&["update-ref", "refs/remotes/origin/main", &head]);
22839 let evidence = crate::pr_outcome::summary_evidence(
22840 &runner.state,
22841 crate::pr_outcome::FinalHead::At(&head),
22842 );
22843 let generated = &runner.state.candidates[0].summary;
22844 runner.state.change_summary = Some(crate::run::ChangeSummary {
22845 base: head.clone(),
22846 head,
22847 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22848 title: summary_title(generated).unwrap(),
22849 body: summary_without_title(generated),
22850 });
22851 }
22852
22853 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22854 let mut runner = runner_at(RunStatus::Gating);
22855 runner.state = state_with_summary("add retries", JA_SUMMARY);
22856 if review_only {
22857 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22858 }
22859 runner.state.repo = dir.to_owned();
22860 runner.state.candidates[0].worktree = dir.to_owned();
22861 runner.state.config.agents = agents;
22862 seed_public_summary(&mut runner);
22863 runner
22864 }
22865
22866 fn gate_questions(runner: &Runner) -> usize {
22867 ask::Questions::open()
22868 .list()
22869 .into_iter()
22870 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22871 .count()
22872 }
22873
22874 #[tokio::test]
22875 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22876 crate::run::pin_test_home();
22877 let dir = tempfile::tempdir().unwrap();
22878 let mut runner = rewrite_runner(
22879 dir.path(),
22880 true,
22881 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22882 );
22883 runner.state.config.roles.language_judge =
22884 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22885 let winner = runner.state.candidates[0].clone();
22886 let msg = runner
22887 .guarded_pr_message(&winner, None, true)
22888 .await
22889 .unwrap()
22890 .expect("an English rewrite posts without asking");
22891 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22892 assert!(msg.body.contains("<details>kept</details>"));
22893 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22894 assert_eq!(gate_questions(&runner), 0);
22895 assert!(runner.state.github_text.is_none());
22896 assert!(!runner.state.seats.contains_key("impl-A"));
22897 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22898 }
22899
22900 #[tokio::test]
22901 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22902 crate::run::pin_test_home();
22903 let dir = tempfile::tempdir().unwrap();
22904 let mut runner = rewrite_runner(
22905 dir.path(),
22906 false,
22907 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22908 );
22909 runner.state.config.roles.language_judge =
22910 Some(crate::config::AgentChoice::One("beta".into()));
22911 runner.state.seats.insert(
22912 "impl-A".into(),
22913 SeatState::new("impl-A", "alpha", runner.state.seed),
22914 );
22915 let winner = runner.state.candidates[0].clone();
22916 let msg = runner
22917 .guarded_pr_message(&winner, None, true)
22918 .await
22919 .unwrap()
22920 .expect("rewritten");
22921 assert!(msg.body.contains("<details>kept</details>"));
22922 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22923 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22924 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22925 }
22926
22927 #[tokio::test]
22928 async fn rewrite_reuses_a_resumable_implementer_session() {
22929 crate::run::pin_test_home();
22930 let dir = tempfile::tempdir().unwrap();
22931 let mut runner = rewrite_runner(
22932 dir.path(),
22933 false,
22934 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22935 );
22936 runner.state.config.roles.language_judge =
22937 Some(crate::config::AgentChoice::One("beta".into()));
22938 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22939 seat.turns = 1;
22940 runner.state.seats.insert("impl-A".into(), seat);
22941 let winner = runner.state.candidates[0].clone();
22942 runner
22943 .guarded_pr_message(&winner, None, true)
22944 .await
22945 .unwrap()
22946 .expect("rewritten");
22947 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22948 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22949 }
22950
22951 #[tokio::test]
22952 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22953 crate::run::pin_test_home();
22956 let dir = tempfile::tempdir().unwrap();
22957 let mut exhausted = spec("alpha");
22958 exhausted.command = vec!["false".into()];
22959 let mut runner = rewrite_runner(
22960 dir.path(),
22961 false,
22962 vec![exhausted, scripted("beta", EN_REWRITE)],
22963 );
22964 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22966 "alpha".into(),
22967 "beta".into(),
22968 ]));
22969 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22970 seat.turns = 1;
22971 runner.state.seats.insert("impl-A".into(), seat);
22972 let winner = runner.state.candidates[0].clone();
22973 let msg = runner
22974 .guarded_pr_message(&winner, None, true)
22975 .await
22976 .unwrap()
22977 .expect("the role rewrites after the implementer fails");
22978 assert!(msg.body.contains("`src/net.rs`"));
22979 assert_eq!(
22980 runner.state.seats["impl-A"].turns, 2,
22981 "the implementer was tried once"
22982 );
22983 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22984 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22985 }
22986
22987 #[tokio::test]
22988 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22989 crate::run::pin_test_home();
22990 let dir = tempfile::tempdir().unwrap();
22991 let mut broken = spec("bad");
22992 broken.command = vec!["false".into()];
22993 let mut runner = rewrite_runner(
22994 dir.path(),
22995 true,
22996 vec![
22997 broken,
22998 scripted("noformat", "Add retries."),
22999 scripted("good", EN_REWRITE),
23000 ],
23001 );
23002 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23003 "bad".into(),
23004 "noformat".into(),
23005 "good".into(),
23006 ]));
23007 let winner = runner.state.candidates[0].clone();
23008 let msg = runner
23009 .guarded_pr_message(&winner, None, true)
23010 .await
23011 .unwrap()
23012 .expect("the third agent rewrites");
23013 assert!(msg.body.contains("`src/net.rs`"));
23014 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23015 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23016 }
23017
23018 #[tokio::test]
23019 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23020 crate::run::pin_test_home();
23021 for (agents, role, saved) in [
23023 (
23024 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23025 Some("beta"),
23026 true,
23027 ),
23028 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23029 (Vec::new(), None, false),
23030 ] {
23031 let dir = tempfile::tempdir().unwrap();
23032 let mut runner = rewrite_runner(dir.path(), true, agents);
23033 runner.state.config.roles.language_judge =
23034 role.map(|r| crate::config::AgentChoice::One(r.into()));
23035 let winner = runner.state.candidates[0].clone();
23036 let first = runner
23037 .guarded_pr_message(&winner, None, true)
23038 .await
23039 .unwrap();
23040 assert!(first.is_none(), "withheld text parks the run");
23041 assert!(runner.state.parked);
23042 assert_eq!(gate_questions(&runner), 1);
23043 let g = runner.state.github_text.as_ref().unwrap();
23044 assert_eq!(g.rewrite.is_some(), saved);
23045 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23046 }
23047 }
23048
23049 #[tokio::test]
23050 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23051 crate::run::pin_test_home();
23052 let dir = tempfile::tempdir().unwrap();
23053 let mut runner = runner_at(RunStatus::Gating);
23054 runner.state =
23055 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23056 runner.state.repo = dir.path().to_owned();
23057 runner.state.candidates[0].worktree = dir.path().to_owned();
23058 let mut author = spec("alpha");
23059 author.command = vec![
23060 "sh".into(),
23061 "-c".into(),
23062 "printf '%s' \"$REWRITE_REPLY\"".into(),
23063 ];
23064 author.env.insert(
23065 "REWRITE_REPLY".into(),
23066 format!(
23067 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23068 "English sentence. ".repeat(600)
23069 ),
23070 );
23071 runner.state.config.agents = vec![author];
23072 seed_public_summary(&mut runner);
23073 let winner = runner.state.candidates[0].clone();
23074 assert!(
23075 runner
23076 .guarded_pr_message(&winner, None, true)
23077 .await
23078 .unwrap()
23079 .is_none()
23080 );
23081 let record = runner.state.github_text.clone().unwrap();
23082 let rewrite = record.rewrite.clone().unwrap();
23083 assert_eq!(rewrite.failed, ["body-language"]);
23084 assert!(
23085 rewrite
23086 .body
23087 .ends_with(&format!("magi:run/{}", runner.state.id))
23088 );
23089 let store = ask::Questions::open();
23090 let q = store.get(record.question.as_ref().unwrap()).unwrap();
23091 assert!(
23092 q.choices
23093 .contains(&crate::github_text::USE_REWRITE.to_owned())
23094 );
23095 for label in [
23096 "ORIGINAL",
23097 "FALLBACK",
23098 "REWRITE",
23099 "Rules still failed: body-language",
23100 "truncated",
23101 ] {
23102 assert!(q.detail.contains(label), "{label}");
23103 }
23104 let artifact = std::fs::read_to_string(crate::run::artifact_path(
23105 &runner.state,
23106 &crate::github_text::artifact_name(&record.fingerprint),
23107 ))
23108 .unwrap();
23109 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23110 runner.state =
23112 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23113 store
23114 .update(&q.id, |q| {
23115 q.say("まだ日本語")?;
23116 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23117 })
23118 .unwrap();
23119 assert!(
23120 runner
23121 .guarded_pr_message(&winner, None, true)
23122 .await
23123 .unwrap()
23124 .is_none()
23125 );
23126 let retry = runner.state.github_text.as_ref().unwrap();
23127 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23128 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23129 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23130 assert!(
23131 q.posting_comparison
23132 .as_ref()
23133 .unwrap()
23134 .matches_detail(&q.detail)
23135 );
23136 store
23137 .update(&q.id, |q| {
23138 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23139 })
23140 .unwrap();
23141 let message = runner
23142 .guarded_pr_message(&winner, None, true)
23143 .await
23144 .unwrap()
23145 .unwrap();
23146 assert_eq!(message.title, rewrite.title);
23147 assert_eq!(message.body, rewrite.body);
23148 assert_eq!(
23149 runner.state.github_text.as_ref().unwrap().choice,
23150 Some(GithubTextChoice::Rewrite)
23151 );
23152 let again = runner
23153 .guarded_pr_message(&winner, None, true)
23154 .await
23155 .unwrap()
23156 .unwrap();
23157 assert_eq!(again.title, message.title);
23158 assert_eq!(again.body, message.body);
23159 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23160 assert!(
23161 !runner
23162 .state
23163 .events
23164 .iter()
23165 .any(|e| e.message.contains(&rewrite.title)
23166 || e.message.contains("English sentence."))
23167 );
23168 }
23169
23170 #[tokio::test]
23171 async fn historical_posting_reply_never_expires_into_fallback() {
23172 crate::run::pin_test_home();
23173 let dir = tempfile::tempdir().unwrap();
23174 let mut runner = runner_at(RunStatus::Gating);
23175 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23176 runner.state.repo = dir.path().to_owned();
23177 runner.state.candidates[0].worktree = dir.path().to_owned();
23178 seed_public_summary(&mut runner);
23179 let winner = runner.state.candidates[0].clone();
23180 assert!(
23181 runner
23182 .guarded_pr_message(&winner, None, true)
23183 .await
23184 .unwrap()
23185 .is_none()
23186 );
23187 let id = runner
23188 .state
23189 .github_text
23190 .as_ref()
23191 .unwrap()
23192 .question
23193 .clone()
23194 .unwrap();
23195 let store = ask::Questions::open();
23196 let mut q = store.get(&id).unwrap();
23197 q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23198 q.answer(ask::Answer::Choice(
23199 crate::github_text::POST_ORIGINAL.into(),
23200 ))
23201 .unwrap();
23202 q.answer_delivered = true;
23203 store.put(&mut q).unwrap();
23204 let source = std::fs::read(store.path_of(&id)).unwrap();
23205 let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23206 std::fs::create_dir_all(&migration).unwrap();
23207 std::fs::write(
23208 migration.join("state.json"),
23209 serde_json::to_vec(&serde_json::json!({
23210 "schema": 1, "sources": [id], "pending": [id]
23211 }))
23212 .unwrap(),
23213 )
23214 .unwrap();
23215 for _ in 0..2 {
23216 assert!(
23217 runner
23218 .guarded_pr_message(&winner, None, true)
23219 .await
23220 .unwrap()
23221 .is_none()
23222 );
23223 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23224 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23225 }
23226 let home = crate::run::home();
23227 crate::task_chat::sweep(
23228 &crate::queue::Queue::at(home.join("queue")),
23229 &crate::talk::Talks::at(home.join("talks")),
23230 &store,
23231 &home,
23232 );
23233 let outbox = crate::task_inquiry::Outbox::at(&home);
23234 outbox
23235 .update_decision(&id, |d| {
23236 d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23237 d.source_answer_at = q.answered_at;
23238 Ok(true)
23239 })
23240 .unwrap();
23241 outbox
23242 .settle(
23243 &id,
23244 "owner-chat",
23245 "Owner confirms no publication remains necessary",
23246 None,
23247 )
23248 .unwrap();
23249 assert!(
23250 runner
23251 .guarded_pr_message(&winner, None, true)
23252 .await
23253 .unwrap()
23254 .is_none()
23255 );
23256 assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23257 assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23258 }
23259
23260 #[tokio::test]
23261 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23262 crate::run::pin_test_home();
23263 let dir = tempfile::tempdir().unwrap();
23264 let mut runner = runner_at(RunStatus::Gating);
23265 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23266 runner.state.repo = dir.path().to_owned();
23267 runner.state.candidates[0].worktree = dir.path().to_owned();
23268 seed_public_summary(&mut runner);
23269 let winner = runner.state.candidates[0].clone();
23270 assert!(
23271 runner
23272 .guarded_pr_message(&winner, None, true)
23273 .await
23274 .unwrap()
23275 .is_none()
23276 );
23277 let record = runner.state.github_text.as_ref().unwrap();
23278 assert!(record.rewrite.is_none());
23279 let mut old = serde_json::to_value(record).unwrap();
23281 old.as_object_mut().unwrap().remove("rewrite");
23282 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23283 assert!(old.rewrite.is_none());
23284 let id = record.question.clone().unwrap();
23285 ask::Questions::open()
23286 .update(&id, |q| {
23287 q.choices.push(crate::github_text::USE_REWRITE.into());
23288 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23289 })
23290 .unwrap();
23291 let message = runner
23292 .guarded_pr_message(&winner, None, true)
23293 .await
23294 .unwrap()
23295 .unwrap();
23296 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23297 assert_eq!(
23298 runner.state.github_text.as_ref().unwrap().choice,
23299 Some(GithubTextChoice::Fallback)
23300 );
23301 }
23302
23303 #[tokio::test]
23304 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23305 crate::run::pin_test_home();
23306 let dir = tempfile::tempdir().unwrap();
23307 let mut runner = runner_at(RunStatus::Gating);
23308 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23309 runner.state.repo = dir.path().to_owned();
23310 runner.state.candidates[0].worktree = dir.path().to_owned();
23311 seed_public_summary(&mut runner);
23312 let winner = runner.state.candidates[0].clone();
23313 assert!(
23314 runner
23315 .guarded_pr_message(&winner, None, true)
23316 .await
23317 .unwrap()
23318 .is_none()
23319 );
23320 let store = ask::Questions::open();
23321 let find = |run: &str| {
23322 store
23323 .list()
23324 .into_iter()
23325 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23326 .unwrap()
23327 };
23328 let run = runner.state.id.clone();
23329 let q = find(&run);
23330 let fp = runner
23331 .state
23332 .github_text
23333 .as_ref()
23334 .unwrap()
23335 .fingerprint
23336 .clone();
23337 let name = crate::github_text::artifact_name(&fp);
23338 let path = crate::run::artifact_path(&runner.state, &name);
23339 assert!(q.detail.contains(&path.display().to_string()));
23340 assert!(q.detail.contains("title-language: title"));
23341 let saved = std::fs::read_to_string(&path).expect("artifact written");
23342 assert!(saved.contains("日本語のタイトル"));
23343 assert!(q.cwd.is_none());
23344 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23345 let say = |id: &str, text: &str, choice: &str| {
23347 store
23348 .update(id, |q| {
23349 q.thread.push(ask::Turn {
23350 who: ask::Who::Operator,
23351 body: text.to_owned(),
23352 at: jiff::Timestamp::now(),
23353 note: None,
23354 });
23355 q.answer(ask::Answer::Choice(choice.to_owned()))
23356 })
23357 .unwrap();
23358 };
23359 say(&q.id, "まだ日本語", "use my text");
23360 assert!(
23361 runner
23362 .guarded_pr_message(&winner, None, true)
23363 .await
23364 .unwrap()
23365 .is_none()
23366 );
23367 let second = find(&run);
23368 assert_ne!(second.id, q.id);
23369 assert!(second.detail.contains("did not pass"));
23370 assert!(second.detail.contains("日本語のタイトル"));
23373 assert!(!second.detail.contains("まだ日本語"));
23374 assert!(second.detail.contains(&path.display().to_string()));
23375 say(&second.id, "fix: retry failed requests", "use my text");
23376 let message = runner
23377 .guarded_pr_message(&winner, None, true)
23378 .await
23379 .unwrap()
23380 .expect("a vetted replacement is posted");
23381 assert_eq!(message.title, "fix: retry failed requests");
23382 assert_eq!(
23383 runner
23384 .state
23385 .github_text
23386 .as_ref()
23387 .unwrap()
23388 .chosen_title
23389 .as_deref(),
23390 Some("fix: retry failed requests")
23391 );
23392 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23393 let again = runner
23395 .guarded_pr_message(&winner, None, true)
23396 .await
23397 .unwrap()
23398 .unwrap();
23399 assert_eq!(again.title, message.title);
23400 assert_eq!(
23401 store
23402 .list()
23403 .into_iter()
23404 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23405 .count(),
23406 2
23407 );
23408 }
23409
23410 #[test]
23411 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23412 let mut state = state_with_summary(
23416 "add retries",
23417 "TITLE: fix(web): batch reads\n- reads run.json once",
23418 );
23419 state.config.graph.language = "ja".to_owned();
23420 let m = pr_message(&state, 'A');
23421 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23422 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23423
23424 let task = "今回やってほしいこと: results projector を直す";
23427 let mut state = state_with_summary(task, "- no title line");
23428 state.config.graph.language = "ja".to_owned();
23429 let m = pr_message(&state, 'A');
23430 assert_eq!(
23431 m.title,
23432 format!("chore: land candidate A of run {}", state.id)
23433 );
23434 assert!(
23435 m.body.contains(&format!(
23436 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23437 )),
23438 "{}",
23439 m.body
23440 );
23441 }
23442
23443 #[test]
23444 fn pr_message_scrubs_home_paths_and_addresses() {
23445 let state = state_with_summary(
23446 "fix it in /Users/someone/src/x",
23447 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23448 );
23449 let m = pr_message(&state, 'A');
23450 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23451 assert!(!m.body.contains(leak), "{}", m.body);
23452 }
23453 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23454 }
23455
23456 #[test]
23457 fn pr_message_survives_a_task_that_closes_details() {
23458 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23459 let m = pr_message(&state, 'A');
23460 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23461 }
23462
23463 #[test]
23464 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23465 let cmd = manual_merge_command(
23466 MergeStyle::Squash,
23467 Path::new("/repo"),
23468 "b",
23469 "fix: \"quoted\" $(x) `y`\n\nbody",
23470 );
23471 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23472 }
23473
23474 #[test]
23475 fn manual_merge_command_matches_the_configured_style() {
23476 let repo = Path::new("/repo");
23477 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23478
23479 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23480 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23481
23482 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23483 assert_eq!(
23484 squash,
23485 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23486 \"Merge magi run 0832 (candidate A)\""
23487 );
23488
23489 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23490 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23491 }
23492
23493 #[test]
23494 fn a_nudge_gets_a_quarter_of_the_budget() {
23495 assert_eq!(retry_budget(secs(1200), true), secs(300));
23497 assert_eq!(retry_budget(secs(3600), true), secs(900));
23498 }
23499
23500 #[test]
23501 fn a_resent_prompt_keeps_the_whole_budget() {
23502 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23505 assert_eq!(retry_budget(secs(60), false), secs(60));
23506 }
23507
23508 #[test]
23509 fn the_floor_never_exceeds_the_original_budget() {
23510 assert_eq!(retry_budget(secs(60), true), secs(60));
23514 assert_eq!(retry_budget(secs(480), true), secs(120));
23515 assert_eq!(retry_budget(secs(0), true), secs(0));
23516 }
23517
23518 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23519 agent::CommandEvidence {
23520 id: "item1".to_owned(),
23521 description: "cargo test".to_owned(),
23522 exit_code,
23523 result_summary: String::new(),
23524 source: "codex".to_owned(),
23525 }
23526 }
23527
23528 #[test]
23529 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23530 assert!(!has_unconfirmed_command(&[]));
23534 }
23535
23536 #[test]
23537 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23538 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23542 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23543 assert!(!has_unconfirmed_command(&[
23544 evidence(Some(0)),
23545 evidence(Some(101))
23546 ]));
23547 }
23548
23549 #[test]
23550 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23551 assert!(has_unconfirmed_command(&[
23552 evidence(Some(0)),
23553 evidence(None)
23554 ]));
23555 }
23556
23557 #[test]
23558 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23559 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23560 assert_eq!(
23561 verified_noop_claim(true, &[], text).as_deref(),
23562 Some("already fixed by b32cfc4, on main.")
23563 );
23564 }
23565
23566 #[test]
23567 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23568 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23571 assert!(verified_noop_claim(false, &[], text).is_none());
23572 }
23573
23574 #[test]
23575 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23576 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23577 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23578 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23580 }
23581
23582 #[test]
23583 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23584 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23585 }
23586
23587 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23590 runner.state.candidates = shape
23591 .iter()
23592 .enumerate()
23593 .map(|(i, &(empty, verified))| Candidate {
23594 index: i,
23595 label: (b'A' + i as u8) as char,
23596 agent: "sonnet".to_owned(),
23597 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23598 worktree: PathBuf::from(format!("/wt/{i}")),
23599 summary: String::new(),
23600 stat: String::new(),
23601 files: 0,
23602 commits: 0,
23603 empty,
23604 failed: None,
23605 verified_noop: verified.map(str::to_owned),
23606 duration_ms: 0,
23607 folded: false,
23608 })
23609 .collect();
23610 }
23611
23612 #[test]
23613 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23614 ask_test_home();
23615 let mut runner = runner_at(RunStatus::Implementing);
23616 set_candidates(
23617 &mut runner,
23618 &[
23619 (true, Some("already on main at b32cfc4")),
23620 (true, Some("same fix, see the existing test")),
23621 ],
23622 );
23623
23624 runner
23625 .after_implement()
23626 .expect("a verified no-op is not an error");
23627
23628 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23629 }
23630
23631 #[test]
23632 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23633 ask_test_home();
23634 let mut runner = runner_at(RunStatus::Implementing);
23635 set_candidates(
23639 &mut runner,
23640 &[(true, Some("already on main at b32cfc4")), (true, None)],
23641 );
23642
23643 let err = runner
23644 .after_implement()
23645 .expect_err("an unverified empty candidate must still fail the run");
23646
23647 assert!(
23648 err.to_string().contains("no candidate produced a change"),
23649 "{err}"
23650 );
23651 assert_eq!(runner.state.status, RunStatus::Failed);
23652 }
23653
23654 #[test]
23655 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23656 ask_test_home();
23657 let mut runner = runner_at(RunStatus::Implementing);
23658 set_candidates(&mut runner, &[(true, None), (true, None)]);
23659
23660 let err = runner
23661 .after_implement()
23662 .expect_err("no candidate declared anything; this is an ordinary failure");
23663
23664 assert!(
23665 err.to_string().contains("no candidate produced a change"),
23666 "{err}"
23667 );
23668 assert_eq!(runner.state.status, RunStatus::Failed);
23669 }
23670}
23671
23672#[derive(Debug)]
23675struct GraftRestoreFailed(String);
23676
23677impl std::fmt::Display for GraftRestoreFailed {
23678 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23679 write!(
23680 f,
23681 "could not restore the winner to its pre-graft commit: {}",
23682 self.0
23683 )
23684 }
23685}
23686
23687impl std::error::Error for GraftRestoreFailed {}
23688
23689#[cfg(test)]
23693mod resource_wait_tests {
23694 use super::*;
23695 use std::collections::BTreeMap;
23696
23697 fn sh(id: &str, script: &str) -> AgentSpec {
23698 AgentSpec {
23699 id: id.to_owned(),
23700 kind: crate::config::AgentKind::Command,
23701 model: None,
23702 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23703 extra_args: Vec::new(),
23704 env: BTreeMap::new(),
23705 prompt_delivery: None,
23706 max_turn_secs: None,
23707 }
23708 }
23709
23710 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23711 SeatJob {
23712 seat: SeatState::new("impl-A", &spec.id, 1),
23713 spec,
23714 cwd: dir.to_path_buf(),
23715 prompt: "do it".to_owned(),
23716 timeout,
23717 allow_write: true,
23718 sessions: false,
23719 artifacts: dir.join("art"),
23720 stem: "impl-A".to_owned(),
23721 handover: None,
23722 }
23723 }
23724
23725 fn state(resource_wait_secs: u64) -> RunState {
23726 let mut config = Config::default();
23727 config.graph.timeout_resource_wait = resource_wait_secs;
23728 RunState::new(
23729 PathBuf::from("/repo"),
23730 "main".to_owned(),
23731 "abc1234".to_owned(),
23732 "task".to_owned(),
23733 config,
23734 )
23735 }
23736
23737 fn hold(cache: &Path) -> crate::cache::Guard {
23738 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23739 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23740 crate::cache::AcquireOutcome::Acquired(g) => g,
23741 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23742 }
23743 }
23744
23745 async fn run(
23746 jobs: Vec<SeatJob>,
23747 sem: Arc<Semaphore>,
23748 cache: Option<&Path>,
23749 st: &mut RunState,
23750 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23751 let prompts = Prompts::default();
23752 let ctx = WaveCtx {
23753 run: "r",
23754 node: "implement",
23755 prompts: &prompts,
23756 cache,
23757 round: None,
23758 carry_seats: false,
23759 pauses: None,
23760 };
23761 wave(jobs, sem, &ctx, st, 0).await
23762 }
23763
23764 #[tokio::test]
23765 async fn verification_cache_wait_preserves_the_command_budget() {
23766 crate::run::pin_test_home();
23767 let tmp = tempfile::tempdir().unwrap();
23768 let cache = tmp.path().join("target");
23769 let guard = hold(&cache);
23770 let release = tokio::spawn(async move {
23771 tokio::time::sleep(Duration::from_millis(100)).await;
23772 guard.release();
23773 });
23774 let mut st = state(10);
23775 let budget = Duration::from_millis(50);
23776 let wait = repository_wait(&st, None);
23777 let (outcomes, _) = with_cache_lease(
23778 &mut st,
23779 Some(&cache),
23780 "gate",
23781 "gate",
23782 tmp.path(),
23783 "abc1234",
23784 budget,
23785 "test gate",
23786 &wait,
23787 |_, available| async move {
23788 assert_eq!(
23789 available, budget,
23790 "contention must not shorten verification"
23791 );
23792 (Vec::new(), false, Vec::new())
23793 },
23794 )
23795 .await
23796 .unwrap();
23797 release.await.unwrap();
23798 assert!(outcomes.is_empty(), "{outcomes:?}");
23799 assert!(
23800 st.events
23801 .iter()
23802 .any(|event| event.message.contains("waiting for"))
23803 );
23804 assert!(st.resource_wait.is_none());
23805 hold(&cache).release();
23807 }
23808
23809 #[tokio::test]
23810 async fn verification_cache_wait_expires_without_starting_commands() {
23811 crate::run::pin_test_home();
23812 let tmp = tempfile::tempdir().unwrap();
23813 let cache = tmp.path().join("target");
23814 let _held = hold(&cache);
23815 let mut st = state(0);
23816 let wait = repository_wait(&st, None);
23817 let (outcomes, _) = with_cache_lease(
23818 &mut st,
23819 Some(&cache),
23820 "gate",
23821 "gate",
23822 tmp.path(),
23823 "abc1234",
23824 Duration::from_secs(60),
23825 "test gate",
23826 &wait,
23827 |_, _| async { panic!("commands must not start without the cache lease") },
23828 )
23829 .await
23830 .unwrap();
23831 assert_eq!(outcomes.len(), 1);
23832 assert!(outcomes[0].resource_blocked);
23833 assert!(st.resource_wait.is_none());
23834 assert!(matches!(
23835 st.resource_block.unwrap().cause,
23836 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23837 ));
23838 }
23839
23840 #[tokio::test]
23841 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23842 crate::run::pin_test_home();
23843 let tmp = tempfile::tempdir().unwrap();
23844 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23845 let cache = tmp.path().join("cache");
23846 let mut state = state(21600);
23847 let wait = repository_wait(&state, None);
23848 let (outcomes, _) = with_cache_lease(
23849 &mut state,
23850 Some(&cache),
23851 "gate",
23852 "gate",
23853 tmp.path(),
23854 "head",
23855 Duration::from_secs(60),
23856 "gate",
23857 &wait,
23858 |_, _| async { panic!("freshness failure must not start verification") },
23859 )
23860 .await
23861 .unwrap();
23862 assert_eq!(outcomes.len(), 1);
23863 assert!(outcomes[0].resource_blocked);
23864 let block = state.resource_block.unwrap();
23865 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23866 assert!(!block.description().contains("timed out"));
23867 }
23868
23869 #[tokio::test]
23870 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23871 crate::run::pin_test_home();
23872 let tmp = tempfile::tempdir().unwrap();
23873 let cache = tmp.path().join("cache");
23874 let _held = hold(&cache);
23875 let mut state = state(21600);
23876 let pause = Pause::new();
23877 let interrupt = Pause::new();
23878 let cause = ParkCause::Operator;
23879 pause.park_for(cause.clone());
23880 let prompts = Prompts::default();
23881 let ctx = WaveCtx {
23882 run: "test",
23883 node: "fix",
23884 prompts: &prompts,
23885 cache: Some(&cache),
23886 round: None,
23887 carry_seats: false,
23888 pauses: Some((&pause, &interrupt)),
23889 };
23890 let result = wave(
23891 vec![job(
23892 sh("writer", "exit 99"),
23893 tmp.path(),
23894 Duration::from_secs(1),
23895 )],
23896 Arc::new(Semaphore::new(1)),
23897 &ctx,
23898 &mut state,
23899 0,
23900 )
23901 .await;
23902 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23903 assert_eq!(result[0].1.turns, 0);
23904 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23905 assert_eq!(
23906 state.resource_block.unwrap().cause,
23907 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23908 );
23909 assert!(state.active.is_empty());
23910 }
23911
23912 #[tokio::test]
23913 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23914 crate::run::pin_test_home();
23915 let tmp = tempfile::tempdir().unwrap();
23916 let cache = tmp.path().join("target-a");
23917 let guard = hold(&cache);
23918 tokio::spawn(async move {
23920 tokio::time::sleep(Duration::from_millis(1500)).await;
23921 guard.release();
23922 });
23923 let mut st = state(60);
23924 let results = run(
23925 vec![job(
23926 sh("alpha", "echo ok"),
23927 tmp.path(),
23928 Duration::from_secs(1),
23929 )],
23930 Arc::new(Semaphore::new(1)),
23931 Some(&cache),
23932 &mut st,
23933 )
23934 .await;
23935 assert!(
23936 matches!(results[0].2, AgentOutcome::Ok(_)),
23937 "the wait must not be charged to the agent"
23938 );
23939 assert!(st.resource_block.is_none());
23940 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23941 assert!(st.resource_wait.is_none() && st.active.is_empty());
23942 }
23943
23944 #[tokio::test]
23945 #[cfg(unix)]
23946 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23947 use std::os::fd::AsRawFd as _;
23948 crate::run::pin_test_home();
23949 let tmp = tempfile::tempdir().unwrap();
23950 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23951 let common = tmp.path().join(".git");
23952 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23953 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23954 assert_eq!(
23955 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23956 0
23957 );
23958 let mut st = state(1);
23959 st.config.coordination = crate::config::test_coordination();
23960 let mut reader = job(
23961 sh("reader", "echo must-not-run"),
23962 tmp.path(),
23963 Duration::from_secs(30),
23964 );
23965 reader.allow_write = false;
23966 let results = run(
23967 vec![reader.clone()],
23968 Arc::new(Semaphore::new(1)),
23969 None,
23970 &mut st,
23971 )
23972 .await;
23973 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23974 assert_eq!(results[0].1.turns, 0);
23975 assert!(FailClass::of(&results[0].2).is_none());
23976 assert_eq!(
23977 st.resource_block.as_ref().unwrap().kind,
23978 "repository coordination"
23979 );
23980 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23981 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23982 drop(external);
23983 let editor = crate::repo_lock::SeatLease::for_checkout(
23984 tmp.path(),
23985 true,
23986 crate::config::test_coordination().as_ref(),
23987 )
23988 .await
23989 .unwrap();
23990 let pause = Pause::new();
23991 let interrupt = Pause::new();
23992 let prompts = Prompts::default();
23993 let ctx = WaveCtx {
23994 run: "r",
23995 node: "review",
23996 prompts: &prompts,
23997 cache: None,
23998 round: None,
23999 carry_seats: false,
24000 pauses: Some((&pause, &interrupt)),
24001 };
24002 let (results, ()) = tokio::join!(
24003 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24004 async {
24005 tokio::time::sleep(Duration::from_millis(50)).await;
24006 pause.park();
24007 }
24008 );
24009 assert!(
24010 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24011 );
24012 assert_eq!(results[0].1.turns, 0);
24013 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24014 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24015 drop(editor);
24016 }
24017
24018 #[tokio::test]
24019 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24020 crate::run::pin_test_home();
24021 let tmp = tempfile::tempdir().unwrap();
24022 let cache = tmp.path().join("target-b");
24023 let _held = hold(&cache);
24024 let mut st = state(1);
24025 let results = run(
24026 vec![job(
24027 sh("alpha", "echo ok"),
24028 tmp.path(),
24029 Duration::from_secs(1),
24030 )],
24031 Arc::new(Semaphore::new(1)),
24032 Some(&cache),
24033 &mut st,
24034 )
24035 .await;
24036 let (_, seat, out) = &results[0];
24037 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24038 assert!(wave_blocked(&results));
24039 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24040 assert_eq!(seat.turns, 0, "nothing was asked");
24041 assert!(
24042 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24043 "no agent was spawned"
24044 );
24045 assert!(FailClass::of(out).is_none(), "not a failure class");
24046 let block = st.resource_block.as_ref().expect("recorded for the resume");
24047 assert_eq!(block.kind, "shared build cache");
24048 assert!(matches!(
24049 block.cause,
24050 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24051 ));
24052 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24053 assert!(st.active.is_empty());
24054 }
24055
24056 #[tokio::test]
24057 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24058 crate::run::pin_test_home();
24059 let tmp = tempfile::tempdir().unwrap();
24060 let mut st = state(1);
24061 let results = run(
24062 vec![job(
24063 sh("alpha", "echo ok"),
24064 tmp.path(),
24065 Duration::from_secs(1),
24066 )],
24067 Arc::new(Semaphore::new(0)),
24068 None,
24069 &mut st,
24070 )
24071 .await;
24072 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24073 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24074 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24075 }
24076
24077 #[tokio::test]
24078 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24079 crate::run::pin_test_home();
24080 let tmp = tempfile::tempdir().unwrap();
24081 let mut st = state(60);
24082 let results = run(
24083 vec![job(
24084 sh("alpha", "sleep 30"),
24085 tmp.path(),
24086 Duration::from_secs(1),
24087 )],
24088 Arc::new(Semaphore::new(1)),
24089 None,
24090 &mut st,
24091 )
24092 .await;
24093 let out = &results[0].2;
24094 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24095 let class = FailClass::of(out).expect("a failure");
24096 assert_eq!(class, FailClass::Timeout);
24097 assert!(should_hand_over(None, &class), "existing handover policy");
24098 assert!(st.resource_block.is_none());
24099 }
24100}