1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 if state.startup.as_ref().is_some_and(|s| !s.complete)
1470 || (state.status == RunStatus::Prep
1471 && state.instruction.trim().is_empty()
1472 && state.candidates.is_empty()
1473 && state.prep_complete == Some(false))
1474 {
1475 bail!(
1476 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477 );
1478 }
1479 crate::pr_head::ensure_owner(&state)?;
1480 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481 bail!(
1482 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483 );
1484 }
1485 if let Some(to) = &state.released_to {
1486 bail!(
1487 "run {} cannot be resumed: its worktree was released to run {}",
1488 state.short(),
1489 crate::run::short_of(to)
1490 );
1491 }
1492 let roles = state.config.resolve_roles()?;
1493 if let Some(id) = &state.config.graph.selected_implementer
1494 && state.candidates.iter().any(|c| c.agent != *id)
1495 {
1496 bail!(
1497 "replacement implementer pin does not match the saved candidates; task must stay held"
1498 );
1499 }
1500 let max_parallel = state.config.graph.max_parallel.max(1);
1501 Ok(Self {
1502 state,
1503 roles,
1504 sem: Arc::new(Semaphore::new(max_parallel)),
1505 pause: Pause::new(),
1506 interrupt: Pause::new(),
1507 })
1508 }
1509
1510 pub async fn execute(&mut self) -> Result<()> {
1517 anyhow::ensure!(
1518 self.state.startup.as_ref().is_none_or(|s| s.complete),
1519 "startup incomplete; inspect retained evidence before explicit recovery"
1520 );
1521 if let Some(selection) = &self.state.pr_recovery
1522 && (selection.source_proof.is_some() || selection.source_run.is_some())
1523 {
1524 Box::pin(selection.verify_source(&self.state.repo)).await?;
1525 }
1526 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529 || self.state.land_approval.is_some()
1530 || self.state.merge_decision.is_some()
1531 || !self.state.merge_repairs.is_empty()
1532 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533 {
1534 let claim = FixClaim::acquire(&self.state.dir())?;
1535 let disk = RunState::load(&self.state.id)?;
1536 let checkpoint = |s: &RunState| {
1537 (
1538 s.merge_repairs.len(),
1539 s.merge_repairs
1540 .last()
1541 .map(|r| (r.question.clone(), r.phase)),
1542 s.land_approval
1543 .as_ref()
1544 .map(|a| (a.question.clone(), a.head.clone())),
1545 )
1546 };
1547 if checkpoint(&disk) != checkpoint(&self.state) {
1548 bail!(
1549 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550 );
1551 }
1552 Some(claim)
1553 } else {
1554 None
1555 };
1556 let mut execution = self.claim_driver().await?;
1557 let runner = &mut *execution.runner;
1558 let wait = runner.repository_wait();
1559 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562 let result = match result {
1563 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564 record_repository_block(&mut runner.state, &e);
1565 runner.park_here()?;
1566 Ok(())
1567 }
1568 other => other,
1569 };
1570 execution.driver.finish(&mut runner.state).await?;
1571 let ended = if result.is_err() {
1572 Some({
1573 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574 if let Err(error) = &result {
1575 notice.message = format!(
1576 "Run {} stopped with an error: {}",
1577 runner.state.id,
1578 error.to_string().lines().next().unwrap_or("unknown error")
1579 );
1580 }
1581 notice
1582 })
1583 } else {
1584 crate::notices::run_ended(&runner.state)
1585 };
1586 if let Some(notice) = ended {
1587 crate::notices::raise(notice);
1588 }
1589 result
1590 }
1591
1592 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594 }
1595
1596 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597 let wait = self.repository_wait();
1598 coordinate_operation(&mut self.state, cwd, &wait).await
1599 }
1600
1601 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603 let owner = match crate::pr_head::owner_lock_wait(
1604 &crate::run::home(),
1605 &self.state.id,
1606 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607 )
1608 .await
1609 {
1610 Ok(owner) => owner,
1611 Err(error) => {
1612 let detail = format!("worktree maintenance or ownership update: {error:#}");
1613 self.state.resource_block = Some(crate::run::ResourceBlock {
1617 cause: crate::run::ResourceBlockCause::Other,
1618 node: "start".into(),
1619 kind: "worktree maintenance".into(),
1620 detail: detail.clone(),
1621 at: Timestamp::now(),
1622 });
1623 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624 }
1625 };
1626 let disk = RunState::load(&self.state.id)?;
1627 crate::pr_head::ensure_owner(&disk)?;
1628 let claimed =
1629 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630 anyhow::ensure!(
1635 disk.liveness(claimed) != crate::run::Liveness::Live,
1636 "run already has a live driver; wait for its execution to end"
1637 );
1638 if let Some(to) = &disk.released_to {
1639 bail!(
1640 "run {} cannot continue: its worktree was released to run {}",
1641 self.state.short(),
1642 crate::run::short_of(to)
1643 );
1644 }
1645 anyhow::ensure!(
1646 serde_json::to_value(&disk.base_sync_recovery)?
1647 == serde_json::to_value(&self.state.base_sync_recovery)?,
1648 "base recovery changed since this run was loaded; reload the saved run"
1649 );
1650 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651 let mut execution = DriverExecution {
1652 runner: self,
1653 driver,
1654 };
1655 let _owner = owner;
1658 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659 .await?;
1660 let state = &mut execution.runner.state;
1661 state.parked = false;
1666 state.resource_wait = None;
1667 state.resource_block = None;
1668 state.clear_active();
1675 let pid = std::process::id();
1690 state.driver_pid = Some(pid);
1691 state.driver_started_at = crate::proc::process_started_at(pid);
1692 state.driver_exited = false;
1693 state.driver_claimed = Some(true);
1694 state.begin_pass();
1695 execution.driver.bind(state);
1696 state.save()?;
1697 execution.driver.record(state)?;
1698 Ok(execution)
1699 }
1700
1701 async fn execute_granted_repair(&mut self) -> Result<()> {
1704 use crate::repair_budget::{Kind, Phase};
1705 let index = self.state.repair_grants.len() - 1;
1706 let winner = self
1707 .state
1708 .winner()
1709 .cloned()
1710 .context("repair winner missing")?;
1711 loop {
1712 let grant = self.state.repair_grants[index].clone();
1713 match grant.phase {
1714 Phase::Pending => {
1715 if grant.spent >= grant.receipt.request.rounds {
1716 self.state.repair_grants[index].phase = Phase::Complete;
1717 self.state.status = RunStatus::Blocked;
1718 self.state.save()?;
1719 return Ok(());
1720 }
1721 self.state.repair_grants[index].spent += 1;
1724 self.state.repair_grants[index].phase = Phase::Fixing;
1725 self.state.save()?;
1726 match grant.receipt.request.kind {
1727 Kind::Gate => {
1728 let failed = self.state.gate.clone();
1729 let _ = self.gate_fix_round(&winner, &failed).await?;
1730 }
1731 Kind::Review => {
1732 let mut record = self
1733 .state
1734 .reviews
1735 .last()
1736 .cloned()
1737 .context("review failure missing")?;
1738 let round = record.round;
1739 let step = self.run_fix_round(&winner, &mut record).await?;
1740 self.state.reviews[round - 1] = record;
1741 if matches!(step, FixStep::Recorded { report_lost: true }) {
1742 self.state.repair_grants[index].phase = Phase::Complete;
1743 self.state.status = RunStatus::Blocked;
1744 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745 self.state.save()?;
1746 return Ok(());
1747 }
1748 if matches!(step, FixStep::Resource) {
1749 self.stop_for_resource()?;
1750 return Ok(());
1751 }
1752 self.state.save()?;
1753 }
1754 }
1755 if self.resource_halt()? {
1756 return Ok(());
1757 }
1758 }
1759 Phase::Fixing => {
1760 let _coordination = self.coordinate(&winner.worktree).await?;
1763 let rescued = git::rescue_commit(
1764 &winner.worktree,
1765 "magi: preserve interrupted granted repair",
1766 )
1767 .await?;
1768 self.state.note_withheld("repair-grant", &rescued.withheld);
1769 self.state
1772 .verification_history
1773 .push(crate::run::VerificationEvidence {
1774 superseded_by: None,
1775 archived_at: Some(Timestamp::now()),
1776 fixes_spent: Some(self.state.gate_fixes.len()),
1777 gate_head: self.state.gate_head.take(),
1778 gate_ran: std::mem::take(&mut self.state.gate_ran),
1779 gate: std::mem::take(&mut self.state.gate),
1780 e2e: self.state.head_e2e.take(),
1781 });
1782 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783 self.state
1784 .verification_history
1785 .push(crate::run::VerificationEvidence {
1786 superseded_by: None,
1787 archived_at: Some(Timestamp::now()),
1788 fixes_spent: Some(self.state.gate_fixes.len()),
1789 gate_head: None,
1790 gate_ran: false,
1791 gate: Vec::new(),
1792 e2e: Some(e2e),
1793 });
1794 }
1795 self.state.repair_grants[index].review_start = self.state.reviews.len();
1796 self.state.repair_grants[index].head =
1797 git::rev_parse(&winner.worktree, "HEAD").await?;
1798 self.state.repair_grants[index].phase = Phase::Reviewing;
1799 self.state.save()?;
1800 }
1801 Phase::Reviewing => {
1802 self.review_loop_pass().await?;
1803 if self.resource_halt()? {
1804 return Ok(());
1805 }
1806 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807 let clean = self.state.reviews.len() > grant.review_start
1808 && self.state.reviews.last().is_some_and(|r| {
1809 r.head == head
1810 && r.clean
1811 && r.expected > 0
1812 && r.answered == r.expected
1813 && matches!(
1814 r.e2e_status(),
1815 E2eStatus::Passed | E2eStatus::NotConfigured
1816 )
1817 });
1818 if !clean {
1819 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821 == Kind::Review
1822 && grant.spent < grant.receipt.request.rounds
1823 && self.state.reviews.last().is_some_and(|r| {
1824 r.fix.is_none()
1825 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826 }) {
1827 Phase::Pending
1828 } else {
1829 Phase::Complete
1830 };
1831 self.state.status = RunStatus::Blocked;
1832 self.state.event(
1833 "repair-grant",
1834 "fresh review did not clear the repaired head; no publication",
1835 );
1836 self.state.save()?;
1837 if self.state.repair_grants[index].phase == Phase::Pending {
1838 continue;
1839 }
1840 return Ok(());
1841 }
1842 self.state.repair_grants[index].phase = Phase::Gating;
1843 self.state.save()?;
1844 }
1845 Phase::Gating => {
1846 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847 self.state.reconcile_verification_head(&head);
1848 let wait = self.repository_wait();
1849 if self
1850 .state
1851 .reviews
1852 .last()
1853 .is_none_or(|r| r.head != head || !r.clean)
1854 {
1855 self.state.status = RunStatus::Blocked;
1856 self.state.repair_grants[index].phase = Phase::Complete;
1857 self.state.event(
1858 "repair-grant",
1859 "head changed since fresh review; repair held",
1860 );
1861 self.state.save()?;
1862 return Ok(());
1863 }
1864 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865 return Ok(());
1866 }
1867 let outcomes = self.run_gate(&winner).await?;
1868 if verify_inconclusive(&outcomes) {
1869 return Ok(());
1870 }
1871 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872 if after != head {
1873 self.state.repair_grants[index].review_start = self.state.reviews.len();
1874 self.state.repair_grants[index].head = after;
1875 self.state.repair_grants[index].phase = Phase::Reviewing;
1876 self.state.status = RunStatus::Blocked;
1877 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878 self.state.save()?;
1879 return Ok(());
1880 }
1881 let passed = outcomes.iter().all(CommandOutcome::ok);
1882 self.state.repair_grants[index].succeeded = passed;
1883 self.state.gate = outcomes;
1884 self.state.gate_ran = true;
1885 self.state.status = if passed {
1886 RunStatus::Gating
1887 } else {
1888 RunStatus::Blocked
1889 };
1890 self.state.repair_grants[index].phase = if !passed
1891 && grant.receipt.request.kind == Kind::Gate
1892 && grant.spent < grant.receipt.request.rounds
1893 {
1894 Phase::Pending
1895 } else {
1896 Phase::Complete
1897 };
1898 if passed && self.state.config.merge.mode == MergeMode::Pr {
1899 self.state.pr_publication_pending = true;
1901 } else if passed {
1902 self.state.status = RunStatus::Ready;
1903 }
1904 self.state.save()?;
1905 if !passed {
1906 continue;
1907 }
1908 if self.state.config.merge.mode != MergeMode::Pr {
1912 self.state.status = RunStatus::Ready;
1913 self.state.event(
1914 "repair-grant",
1915 "repair verified locally; grant carries no merge authorization",
1916 );
1917 self.state.save()?;
1918 return Ok(());
1919 }
1920 return self.merge().await;
1921 }
1922 Phase::Complete => return Ok(()),
1923 }
1924 }
1925 }
1926
1927 async fn execute_graph(&mut self) -> Result<()> {
1928 let archive_home = crate::run::home().canonicalize()?;
1932 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1933 let repo = self.state.repo.clone();
1934 let _coordination = self.coordinate(&repo).await?;
1935 ensure_setup_config(&mut self.state, &repo).await?;
1936 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1937 }
1938 self.state.configure_recovery_resume()?;
1939 if self
1940 .state
1941 .repair_grants
1942 .last()
1943 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1944 {
1945 return self.execute_granted_repair().await;
1946 }
1947 if self
1948 .state
1949 .repair_grants
1950 .last()
1951 .is_some_and(|g| !g.succeeded)
1952 {
1953 self.state.status = RunStatus::Blocked;
1954 self.state.save()?;
1955 return Ok(());
1956 }
1957 self.persist_candidates().await?;
1960 if self.state.base_sync_blocked() {
1963 self.sync_to_base().await?;
1964 if !self.state.base_sync_blocked()
1965 && self.state.status != RunStatus::AlreadyInBase
1966 && !self.resource_halt()?
1967 {
1968 self.finish_after_tally().await?;
1969 }
1970 return Ok(());
1971 }
1972 if self
1973 .state
1974 .merge_decision
1975 .as_ref()
1976 .is_some_and(|decision| !decision.closed)
1977 && self.state.status != RunStatus::Landing
1978 {
1979 if crate::merge_decision::resumable_hold(&self.state) {
1980 self.state.merge = Some(MergeOutcome {
1981 mode: MergeMode::Pr,
1982 ok: false,
1983 detail: self
1984 .state
1985 .merge_decision
1986 .as_ref()
1987 .unwrap()
1988 .target
1989 .pr
1990 .clone(),
1991 empty: false,
1992 });
1993 self.run_land().await?;
1994 self.settle_questions();
1995 return Ok(());
1996 }
1997 bail!(
1998 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1999 );
2000 }
2001 if self.state.config.graph.selected_implementer.is_some()
2002 && self.state.status == RunStatus::Blocked
2003 && self.state.tally.is_none()
2004 && self.state.candidate_preservation.is_empty()
2005 && self.state.interrupted_implementations.is_empty()
2006 {
2007 bail!(
2008 "selected implementation is held; explicitly release the owning task to retry it"
2009 );
2010 }
2011
2012 if self.state.merge_repairs.last().is_some_and(|r| {
2013 matches!(
2014 r.phase,
2015 crate::run::MergeRepairPhase::Requested
2016 | crate::run::MergeRepairPhase::Fixing
2017 | crate::run::MergeRepairPhase::Reviewing
2018 )
2019 }) {
2020 if self.repair_merge_review().await? {
2021 self.run_land().await?;
2022 }
2023 self.settle_questions();
2024 return Ok(());
2025 }
2026 if self.state.pr_publication_pending {
2029 self.merge().await?;
2030 if self.resource_halt()? {
2031 return Ok(());
2032 }
2033 self.state.save()?;
2034 self.settle_questions();
2035 return Ok(());
2036 }
2037 if self.state.status == RunStatus::Stalled {
2050 if self.recover_stall().await? {
2051 self.finish_after_tally().await?;
2052 } else {
2053 self.state.save()?;
2055 }
2056 return Ok(());
2057 }
2058 if self.state.status == RunStatus::Landing
2068 || crate::handover::landing_checkpoint(&self.state)
2069 {
2070 self.state.land_sync_pending = true;
2071 self.state.save()?;
2072 self.run_land().await?;
2073 self.settle_questions();
2078 return Ok(());
2079 }
2080 if self.state.github_text.is_some()
2085 && self.state.merge.is_none()
2086 && self.state.status == RunStatus::Gating
2087 {
2088 self.merge().await?;
2089 if self.resource_halt()? {
2090 return Ok(());
2091 }
2092 self.state.save()?;
2093 return Ok(());
2094 }
2095 if let Err(e) = self.prep().await {
2096 if e.is::<crate::repo_lock::ResourceBlocked>() {
2097 return Err(e);
2098 }
2099 self.state.status = RunStatus::Blocked;
2100 self.state
2101 .prep_failure
2102 .get_or_insert_with(|| format!("{e:#}"));
2103 self.state
2104 .event("prep", format!("preparation stopped: {e:#}"));
2105 self.state.save()?;
2106 return Err(e);
2107 }
2108 if self.park_here()? {
2109 return Ok(());
2110 }
2111 self.advise().await?;
2112 if self.park_here()? {
2113 return Ok(());
2114 }
2115 self.implement().await?;
2116 if self.park_here()? {
2117 return Ok(());
2118 }
2119 if self.state.status == RunStatus::VerifiedNoop {
2123 return Ok(());
2124 }
2125 self.judge().await?;
2126 if self.park_here()? {
2127 return Ok(());
2128 }
2129 self.deliberate().await?;
2130 if self.park_here()? {
2131 return Ok(());
2132 }
2133 self.vote().await?;
2134 if self.park_here()? {
2135 return Ok(());
2136 }
2137 self.tally()?;
2138 if self.state.status == RunStatus::Stalled {
2143 self.state.save()?;
2147 return Ok(());
2148 }
2149 self.finish_after_tally().await?;
2150 Ok(())
2151 }
2152
2153 fn stop_for_resource(&mut self) -> Result<()> {
2158 if self.pause.parked() || self.interrupt.parked() {
2161 self.park_here()?;
2162 return Ok(());
2163 }
2164 if self.state.resource_block.is_some() {
2165 self.state.status = RunStatus::Blocked;
2166 }
2167 self.state.save()
2168 }
2169
2170 fn resource_halt(&mut self) -> Result<bool> {
2172 if self.state.resource_block.is_none() {
2173 return Ok(false);
2174 }
2175 self.stop_for_resource()?;
2176 Ok(true)
2177 }
2178
2179 fn park_here(&mut self) -> Result<bool> {
2186 let parking = self.pause.parked() || self.interrupt.parked();
2187 if !parking && self.state.resource_block.is_some() {
2188 self.stop_for_resource()?;
2189 return Ok(true);
2190 }
2191 if !self.pause.parked() && !self.interrupt.parked() {
2196 return Ok(false);
2197 }
2198 if self.state.parked {
2199 return Ok(true);
2200 }
2201 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2202 if let Some(block) = &mut self.state.resource_block
2203 && matches!(
2204 block.cause,
2205 crate::run::ResourceBlockCause::Cancelled { .. }
2206 )
2207 {
2208 block.cause = crate::run::ResourceBlockCause::Cancelled {
2209 park: cause.clone(),
2210 };
2211 }
2212 let why = match cause.as_ref().map(ParkCause::label) {
2213 Some(reason) => format!(
2214 "parked after `{}` ({reason}) — resume to carry on from here",
2215 self.state.status.as_str()
2216 ),
2217 None => format!(
2218 "parked after `{}` — resume to carry on from here",
2219 self.state.status.as_str()
2220 ),
2221 };
2222 self.state.event("park", why);
2223 self.state
2224 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2225 self.state.parked = true;
2226 self.state.save()?;
2227 Ok(true)
2228 }
2229
2230 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2232 if self.state.prep_failure.is_none() {
2233 return Ok(());
2234 }
2235 let failed_at = self
2236 .state
2237 .events
2238 .iter()
2239 .rev()
2240 .find(|event| event.node == "prep")
2241 .map_or(self.state.updated_at, |event| event.at);
2242 anyhow::ensure!(
2243 approval_at > failed_at,
2244 "preparation retry requires a new explicit release after the failure"
2245 );
2246 self.state.prep_failure = None;
2247 self.state.prep_complete = Some(false);
2248 self.state
2249 .event("prep-retry", "operator approved retry of preparation");
2250 Ok(())
2251 }
2252
2253 pub fn on_pause(&mut self, pause: Pause) {
2255 self.pause = pause;
2256 }
2257
2258 pub fn watch_interrupt(&mut self, pause: Pause) {
2264 self.interrupt = pause;
2265 }
2266
2267 fn settle_questions(&mut self) {
2287 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2288 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2289 }
2290 }
2291
2292 async fn finish_after_tally(&mut self) -> Result<()> {
2295 self.graft().await?;
2298 if self.resource_halt()? {
2299 return Ok(());
2300 }
2301 self.fold_losers().await?;
2302 self.sync_to_base().await?;
2307 if self.hold_base_sync().await? || self.resource_halt()? {
2308 return Ok(());
2309 }
2310 if self.state.status == RunStatus::AlreadyInBase {
2311 return Ok(());
2312 }
2313 self.review_loop().await?;
2314 if self.resource_halt()? {
2315 return Ok(());
2316 }
2317 self.sync_to_base().await?;
2318 if self.hold_base_sync().await? || self.resource_halt()? {
2319 return Ok(());
2320 }
2321 if self.state.status == RunStatus::AlreadyInBase {
2322 return Ok(());
2323 }
2324 self.gate().await?;
2325 self.merge().await?;
2326 if self.resource_halt()? {
2327 return Ok(());
2328 }
2329 self.state.save()?;
2330 Ok(())
2331 }
2332
2333 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2340 let result = match ensure_setup_config(&mut self.state, repo).await {
2341 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2342 Err(e) => Err(e),
2343 };
2344 if let Err(e) = result {
2345 git::worktree_remove_clean(repo, wt).await.ok();
2346 if in_prep {
2347 self.rollback_prep(repo).await;
2348 }
2349 self.state.status = RunStatus::Blocked;
2350 self.state.event(
2351 "setup",
2352 format!("worktree setup failed in {}: {e:#}", wt.display()),
2353 );
2354 self.state.save()?;
2355 return Err(e);
2356 }
2357 Ok(())
2358 }
2359
2360 async fn rollback_prep(&mut self, repo: &Path) {
2362 let root = self.state.worktree_root();
2363 for c in std::mem::take(&mut self.state.candidates) {
2364 if matches!(
2365 git::worktree_remove_clean(repo, &c.worktree).await,
2366 Ok(true)
2367 ) {
2368 git::branch_delete(repo, &c.branch).await.ok();
2369 } else {
2370 self.state.candidates.push(c);
2371 }
2372 }
2373 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2374 for j in 1..=seats {
2375 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2376 .await
2377 .ok();
2378 }
2379 for k in 1..=self.state.config.graph.advisors {
2380 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2381 .await
2382 .ok();
2383 }
2384 }
2385
2386 async fn prep(&mut self) -> Result<()> {
2387 if let Some(reason) = &self.state.prep_failure {
2388 bail!(
2389 "candidate preparation previously failed: {reason}; correct the source before retrying"
2390 );
2391 }
2392 if self
2393 .state
2394 .prep_complete
2395 .unwrap_or(!self.state.candidates.is_empty())
2396 {
2397 return Ok(());
2398 }
2399 let repo = self.state.repo.clone();
2400 let _coordination = self.coordinate(&repo).await?;
2403 let base = self.state.base_commit.clone();
2404 let plan = self.state.sources.plan(&repo, &base).await?;
2405 self.rollback_prep(&repo).await;
2406 if !self.state.candidates.is_empty() {
2407 bail!(
2408 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2409 );
2410 }
2411 self.state.prep_complete = Some(false);
2412 self.state.status = RunStatus::Prep;
2413 self.state.save()?;
2414 let repo = self.state.repo.clone();
2415 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2416 let root = self.state.worktree_root();
2417 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2418
2419 let hooks_dir = self.state.dir().join("hooks");
2422 if self.state.config.blind.commit_msg_hook {
2423 std::fs::create_dir_all(&hooks_dir)
2424 .with_context(|| format!("create {}", hooks_dir.display()))?;
2425 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2426 let path = hooks_dir.join("commit-msg");
2427 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2428 make_executable(&path)?;
2429 git::acquire_worktree_config(&repo).await?;
2437 self.state.enabled_worktree_config = true;
2438 }
2439
2440 ensure_setup_config(&mut self.state, &repo).await?;
2443
2444 for (index, (spec, label)) in self
2445 .roles
2446 .implementers
2447 .clone()
2448 .into_iter()
2449 .zip(labels)
2450 .enumerate()
2451 {
2452 let branch = self.state.branch_for(label);
2453 let worktree = root.join(format!("cand-{label}"));
2454 self.state.event(
2458 "prep-intent",
2459 serde_json::json!({
2460 "branch": branch,
2461 "worktree": worktree,
2462 "start": start,
2463 "picks": plan.picks,
2464 })
2465 .to_string(),
2466 );
2467 self.state.save()?;
2468 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2469 if self.state.config.blind.commit_msg_hook {
2470 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2471 }
2472 git::local_exclude(&worktree, "/.magi/").await?;
2473 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2474 git::worktree_remove_clean(&repo, &worktree).await.ok();
2477 git::branch_delete(&repo, &branch).await.ok();
2478 self.rollback_prep(&repo).await;
2479 self.state.status = RunStatus::Blocked;
2480 self.state
2481 .event("prep", format!("worktree setup failed: {e:#}"));
2482 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2483 self.state.save()?;
2484 return Err(e);
2485 }
2486 self.state.candidates.push(Candidate {
2487 index,
2488 label,
2489 agent: spec.id.clone(),
2490 branch,
2491 worktree,
2492 summary: String::new(),
2493 stat: String::new(),
2494 files: 0,
2495 commits: 0,
2496 empty: false,
2497 failed: None,
2498 verified_noop: None,
2499 duration_ms: 0,
2500 folded: false,
2501 });
2502 self.state.save()?;
2503 for pick in &plan.picks {
2504 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2505 {
2506 self.state.status = RunStatus::Blocked;
2507 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2508 self.state
2509 .event("prep", format!("cannot apply referenced commit: {e}"));
2510 self.state.save()?;
2511 return Err(e);
2512 }
2513 }
2514 }
2515
2516 for j in 1..=self.roles.judges.len() {
2517 let wt = root.join(format!("judge-{j}"));
2518 if !wt.exists() {
2519 git::worktree_add_detached(&repo, &wt, &base).await?;
2520 self.setup_seat_worktree(&repo, &wt, true).await?;
2521 }
2522 }
2523
2524 if self.state.config.graph.advise {
2532 for k in 1..=self.state.config.graph.advisors {
2533 let wt = root.join(format!("advisor-{k}"));
2534 if !wt.exists() {
2535 git::worktree_add_detached(&repo, &wt, &base).await?;
2536 self.setup_seat_worktree(&repo, &wt, true).await?;
2537 }
2538 }
2539 }
2540
2541 let authors: Vec<&str> = self
2546 .roles
2547 .implementers
2548 .iter()
2549 .map(|a| a.id.as_str())
2550 .collect();
2551 let overlap: Vec<String> = self
2552 .roles
2553 .judges
2554 .iter()
2555 .enumerate()
2556 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2557 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2558 .collect();
2559 if !overlap.is_empty() {
2560 let note = format!(
2561 "{} also authored a candidate; blind, but the panel is less \
2562 independent than {} distinct agents would be",
2563 overlap.join(", "),
2564 self.roles.judges.len()
2565 );
2566 self.state.event("prep", note);
2567 }
2568
2569 self.state.event(
2570 "prep",
2571 format!(
2572 "{} candidates, {} judges, base {} ({})",
2573 self.state.candidates.len(),
2574 self.roles.judges.len(),
2575 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2576 self.state.base_branch
2577 ),
2578 );
2579 self.state.prep_complete = Some(true);
2580 self.state.status = RunStatus::Implementing;
2581 self.state.save()?;
2582 Ok(())
2583 }
2584
2585 async fn advise(&mut self) -> Result<()> {
2618 let implement_untouched = self
2619 .state
2620 .candidates
2621 .iter()
2622 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2623 if !self.state.config.graph.advise || self.state.advise_attempted {
2624 return Ok(());
2625 }
2626 if !implement_untouched {
2627 self.state.event(
2628 "advise",
2629 "skipping the design-deliberation stage: at least one \
2630 candidate already shows implementation progress, so this \
2631 run is past the point the stage exists to run before"
2632 .to_owned(),
2633 );
2634 self.state.advise_attempted = true;
2635 self.state.save()?;
2636 return Ok(());
2637 }
2638 let run_id = self.state.id.clone();
2639 let prompts = self.state.config.prompts.clone();
2640 let instruction = self.state.instruction.clone();
2641 let language = self.state.config.graph.language.clone();
2642 let root = self.state.worktree_root();
2643 let n = self.state.config.graph.advisors;
2644 let where_recorded = self.state.dir().join("run.json");
2645
2646 let seats = match self.state.config.advisors() {
2647 Ok(seats) if !seats.is_empty() => seats,
2648 Ok(_) => {
2649 self.state.event(
2650 "advise",
2651 format!(
2652 "[graph] advisors is 0; skipping the design-deliberation \
2653 stage and continuing without a synthesis brief (see {})",
2654 where_recorded.display()
2655 ),
2656 );
2657 self.state.advise_attempted = true;
2658 self.state.save()?;
2659 return Ok(());
2660 }
2661 Err(e) => {
2662 self.state.event(
2663 "advise",
2664 format!(
2665 "could not resolve advisor seats ({e:#}); continuing \
2666 without a design-deliberation brief (see {})",
2667 where_recorded.display()
2668 ),
2669 );
2670 self.state.advise_attempted = true;
2671 self.state.save()?;
2672 return Ok(());
2673 }
2674 };
2675
2676 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2677 let artifacts = agent::artifacts_dir(&self.state.dir());
2678 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2679
2680 let mut jobs = Vec::new();
2681 for (i, spec) in seats.iter().cloned().enumerate() {
2682 let seat_key = format!("advisor-{}", i + 1);
2683 let seat = self.seat(&seat_key, &spec.id);
2684 jobs.push(SeatJob {
2685 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2686 spec,
2687 seat,
2688 cwd: worktrees[i % worktrees.len()].clone(),
2689 timeout,
2690 allow_write: false,
2691 sessions: false,
2692 artifacts: artifacts.clone(),
2693 stem: seat_key,
2694 handover: None,
2695 });
2696 }
2697
2698 self.state.event(
2699 "advise",
2700 format!(
2701 "{} advisor seat(s) sketching a design in parallel",
2702 jobs.len()
2703 ),
2704 );
2705 let mut quota_losses = Vec::new();
2706 let cache = self.state.config.cache_dir();
2707 let ctx = WaveCtx {
2708 carry_seats: false,
2709 pauses: Some((&self.pause, &self.interrupt)),
2710 run: &run_id,
2711 node: "advise",
2712 prompts: &prompts,
2713 cache: cache.as_deref(),
2714 round: None,
2715 };
2716 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2717 let results = ask_json_wave::<Proposal>(
2718 jobs,
2719 Arc::clone(&self.sem),
2720 self.state.config.graph.retries,
2721 &advisor_roster,
2722 &ctx,
2723 &mut quota_losses,
2724 &mut self.state,
2725 &|p: &Proposal| p.validate(),
2726 )
2727 .await;
2728 if self.state.resource_block.is_some() {
2729 return self.stop_for_resource();
2730 }
2731 self.state.quota.extend(quota_losses);
2732
2733 let mut records = Vec::with_capacity(results.len());
2734 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2735 let agent_id = seat.agent.clone();
2736 self.state.seats.insert(seat.key.clone(), seat);
2737 match res {
2738 Ok((proposal, out)) => {
2739 self.state
2740 .event("advise", format!("advisor-{} proposed a design", i + 1));
2741 records.push(advise::AdvisorRecord::proposed(
2742 i + 1,
2743 agent_id,
2744 proposal,
2745 out.duration_ms,
2746 ));
2747 }
2748 Err(e) => {
2749 self.state.event(
2750 "advise",
2751 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2752 );
2753 records.push(advise::AdvisorRecord::failed(
2754 i + 1,
2755 agent_id,
2756 e.to_string(),
2757 ));
2758 }
2759 }
2760 }
2761
2762 let mut advice = advise::Advice {
2763 records,
2764 synthesis: None,
2765 };
2766 if advice.proposals().is_empty() {
2767 self.state.event(
2768 "advise",
2769 "no advisor produced a usable proposal; continuing without a \
2770 synthesis brief"
2771 .to_owned(),
2772 );
2773 } else {
2774 match self
2775 .synthesize_brief(
2776 &advice,
2777 &instruction,
2778 &language,
2779 &worktrees[0],
2780 &artifacts,
2781 &run_id,
2782 &prompts,
2783 cache.as_deref(),
2784 )
2785 .await
2786 {
2787 Ok(Some(text)) => {
2788 self.state.event(
2789 "advise",
2790 "synthesized a design brief for the implementer".to_owned(),
2791 );
2792 advice.synthesis = Some(text);
2793 }
2794 Ok(None) => {
2795 self.state.event(
2796 "advise",
2797 "the synthesis seat produced nothing usable; continuing \
2798 without a design brief"
2799 .to_owned(),
2800 );
2801 }
2802 Err(e) => {
2803 if e.is::<crate::repo_lock::ResourceBlocked>() {
2804 return Err(e);
2805 }
2806 self.state.event(
2807 "advise",
2808 format!("could not synthesize a design brief: {e:#}"),
2809 );
2810 }
2811 }
2812 }
2813 advise::apply_reflection(&mut advice);
2814
2815 self.state.advice = Some(advice);
2816 self.state.advise_attempted = true;
2817 self.state.save()?;
2818 Ok(())
2819 }
2820
2821 #[allow(clippy::too_many_arguments)]
2833 async fn synthesize_brief(
2834 &mut self,
2835 advice: &advise::Advice,
2836 instruction: &str,
2837 language: &str,
2838 cwd: &Path,
2839 artifacts: &Path,
2840 run_id: &str,
2841 prompts: &Prompts,
2842 cache: Option<&Path>,
2843 ) -> Result<Option<String>> {
2844 let chain = agent::pick_chain(
2845 &self.state.config.agents,
2846 self.state.config.roles.synthesizer.as_ref(),
2847 &agent::installed,
2848 "synthesizer",
2849 )?;
2850 let proposals = advice.proposals();
2851 let mut prompt = prompt::with_overlay(
2852 prompt::synthesize_brief(instruction, &proposals, language),
2853 prompts.overlay("advise"),
2854 );
2855 if cache.is_some() {
2856 prompt.push('\n');
2861 prompt.push_str(&prompt::build_cache_note("advise", false));
2862 }
2863 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2864 let mut last = None;
2869 for (n, spec) in chain.iter().enumerate() {
2870 if n > 0 {
2871 self.state
2872 .event("advise", format!("synthesis falling back to {}", spec.id));
2873 }
2874 let mut seat = self.seat("advise-synthesis", &spec.id);
2875 let wait = self.repository_wait();
2876 let outcome = wait
2877 .scope(agent::invoke(
2878 spec,
2879 &mut seat,
2880 &Invocation {
2881 coordination: self.state.config.coordination.as_ref(),
2882 fetched_base: None,
2883 cwd,
2884 prompt: &prompt,
2885 timeout,
2886 allow_write: false,
2887 sessions: false,
2888 artifacts,
2889 stem: &if n == 0 {
2890 "advise-synthesis".to_owned()
2891 } else {
2892 format!("advise-synthesis-{}", spec.id)
2893 },
2894 run: run_id,
2895 node: "advise",
2896 cache_dir: None,
2897 attachments: &[],
2898 writable: &[],
2899 },
2900 ))
2901 .await;
2902 if outcome.is_ok() {
2903 self.state.seats.insert(seat.key.clone(), seat);
2904 }
2905 let advance = agent::chain_advances(&outcome);
2906 last = Some(outcome);
2907 if !advance {
2908 break;
2909 }
2910 }
2911 let out = last.expect("a chain holds at least one agent")?;
2914 if !out.usable() {
2915 return Ok(None);
2916 }
2917 let text =
2918 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2919 Ok((!text.trim().is_empty()).then_some(text))
2920 }
2921
2922 async fn implement(&mut self) -> Result<()> {
2925 self.persist_candidates().await?;
2926 let run_id = self.state.id.clone();
2931 let prompts = self.state.config.prompts.clone();
2932 let todo: Vec<usize> = self
2933 .state
2934 .candidates
2935 .iter()
2936 .enumerate()
2937 .filter(|(_, c)| {
2938 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2939 && c.failed.is_none()
2940 && !c.empty
2941 })
2942 .map(|(i, _)| i)
2943 .collect();
2944 if todo.is_empty() {
2945 return self.after_implement();
2946 }
2947 self.state.status = RunStatus::Implementing;
2948
2949 let language = self.state.config.graph.language.clone();
2950 let github_language = self.state.config.graph.github_language.clone();
2951 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2952 let sessions = self.state.config.graph.sessions;
2953 let artifacts = agent::artifacts_dir(&self.state.dir());
2954 let brief = self
2958 .state
2959 .advice
2960 .as_ref()
2961 .and_then(|a| a.synthesis.as_deref())
2962 .map(str::to_owned);
2963 let attachments = self.state.attachments.clone();
2964
2965 let mut jobs = Vec::new();
2966 for &i in &todo {
2967 let (index, label, worktree) = {
2968 let c = &self.state.candidates[i];
2969 (c.index, c.label, c.worktree.clone())
2970 };
2971 let spec = self.roles.implementers[index].clone();
2972 let seat_key = format!("impl-{label}");
2973 let seat = self.seat(&seat_key, &spec.id);
2974 let instruction = seeded_instruction(&self.state);
2975 jobs.push(SeatJob {
2976 spec,
2977 seat,
2978 prompt: prompt::implement_in(
2979 &instruction,
2980 &worktree.to_string_lossy(),
2981 &language,
2982 &github_language,
2983 brief.as_deref(),
2984 &attachments,
2985 ),
2986 cwd: worktree,
2987 timeout,
2988 allow_write: true,
2989 sessions,
2990 artifacts: artifacts.clone(),
2991 stem: format!("impl-{label}"),
2992 handover: None,
2993 });
2994 }
2995
2996 self.state.event(
2997 "implement",
2998 format!("{} candidates in parallel", jobs.len()),
2999 );
3000 let mut sent = jobs.clone();
3006 let cache = self.state.config.cache_dir();
3007 let ctx = WaveCtx {
3008 carry_seats: false,
3009 pauses: Some((&self.pause, &self.interrupt)),
3010 run: &run_id,
3011 node: "implement",
3012 prompts: &prompts,
3013 cache: cache.as_deref(),
3014 round: None,
3015 };
3016 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3017 if !wave_blocked(&results) {
3018 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3019 .await;
3020 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3021 .await?;
3022 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3023 .await;
3024 }
3025 if !results.is_empty()
3028 && results
3029 .iter()
3030 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3031 {
3032 let losses: Vec<_> = results
3033 .iter()
3034 .map(|(_, seat, out)| {
3035 let AgentOutcome::Quota(o) = out else {
3036 unreachable!()
3037 };
3038 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3039 })
3040 .collect();
3041 self.state.quota.extend(losses.clone());
3042 for (_, seat, _) in results {
3043 self.state.seats.insert(seat.key.clone(), seat);
3044 }
3045 for &i in &todo {
3046 let label = self.state.candidates[i].label;
3047 self.state.interrupted_implementations.insert(label);
3048 self.state
3049 .candidate_preservation
3050 .insert(label, "machine outage awaiting persistence".into());
3051 }
3052 block_machine_outage(&mut self.state, "implement", &losses);
3055 self.state.save()?;
3056 self.persist_candidates().await?;
3057 for &i in &todo {
3058 self.state
3059 .interrupted_implementations
3060 .insert(self.state.candidates[i].label);
3061 }
3062 return self.stop_for_resource();
3063 }
3064 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3065
3066 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3067 && results
3068 .iter()
3069 .any(|(_, _, out)| FailClass::of(out).is_some());
3070 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3071 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3072 if self
3076 .state
3077 .candidate_preservation
3078 .contains_key(&self.state.candidates[i].label)
3079 {
3080 self.state.seats.insert(seat.key.clone(), seat);
3081 }
3082 continue;
3083 }
3084 let seat_key = seat.key.clone();
3085 let agent = seat.agent.clone();
3094 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3095 self.state.seats.insert(seat.key.clone(), seat);
3096 let label = self.state.candidates[i].label;
3097 self.state.interrupted_implementations.remove(&label);
3098
3099 let (summary, duration, failed, verified_claim) = match out {
3100 AgentOutcome::Ok(o) => {
3101 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3102 let failed = (!o.usable()).then(|| {
3103 if o.timed_out {
3104 "agent timed out".to_owned()
3105 } else {
3106 format!(
3107 "agent exited with {}",
3108 crate::notices::exit_status(o.exit_code)
3109 )
3110 }
3111 });
3112 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3113 (text, o.duration_ms, failed, verified_claim)
3114 }
3115 AgentOutcome::Dropped(o) => {
3121 let why = o
3122 .dropped
3123 .as_ref()
3124 .map(|d| d.why.as_str())
3125 .unwrap_or("the CLI ended the stream without delivering its answer");
3126 (
3127 String::new(),
3128 o.duration_ms,
3129 Some(format!("the CLI dropped the stream ({why})")),
3130 None,
3131 )
3132 }
3133 AgentOutcome::Quota(o) => {
3134 self.state.quota.push(QuotaLoss::from_report(
3135 seat_key,
3136 "implement",
3137 o.quota.as_ref(),
3138 ));
3139 (
3140 String::new(),
3141 o.duration_ms,
3142 Some("rate limited (quota); produced no change".to_owned()),
3143 None,
3144 )
3145 }
3146 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3147 (String::new(), 0, Some(e), None)
3148 }
3149 };
3150
3151 let c = &mut self.state.candidates[i];
3154 if !exhausted_the_fallback_chain {
3155 c.agent = agent;
3156 }
3157 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3158 c.duration_ms = duration;
3159 c.failed = failed;
3160 c.verified_noop = verified_claim;
3161 self.state
3162 .candidate_preservation
3163 .insert(label, "awaiting persistence".to_owned());
3164 }
3165 self.state.save()?;
3166 self.persist_candidates().await?;
3167
3168 if resource_blocked {
3169 return self.stop_for_resource();
3170 }
3171 if selected_failed {
3172 self.state.status = RunStatus::Blocked;
3173 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3174 self.state.save()?;
3175 bail!(
3176 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3177 );
3178 }
3179 self.after_implement()
3180 }
3181
3182 async fn persist_candidates(&mut self) -> Result<()> {
3184 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3185 for label in labels {
3186 let i = self
3187 .state
3188 .candidates
3189 .iter()
3190 .position(|c| c.label == label)
3191 .context("pending candidate is missing")?;
3192 let c = self.state.candidates[i].clone();
3193 let interrupted = self.state.interrupted_implementations.contains(&label);
3194 let result = async {
3195 let wait = crate::repo_lock::WaitPolicy::new(
3196 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3197 || false,
3198 );
3199 let _coordination =
3200 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3201 if let Err(e) = git::snapshot_uncommitted(
3204 &c.worktree,
3205 &self
3206 .state
3207 .dir()
3208 .join("artifacts")
3209 .join(format!("candidate-{label}-recovery")),
3210 )
3211 .await
3212 {
3213 self.state.event(
3214 "preserve",
3215 format!("candidate {label} snapshot incomplete: {e:#}"),
3216 );
3217 }
3218 let rescued = git::rescue_commit(
3219 &c.worktree,
3220 &format!("magi: candidate {label} (uncommitted work)"),
3221 )
3222 .await?;
3223 self.state.note_withheld("implement", &rescued.withheld);
3224 let base = &self.state.base_commit;
3225 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3226 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3227 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3228 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3229 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3230 if c.empty && commits > 0 && !patch.trim().is_empty()
3231 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3232 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3233 || self.state.tally.is_some())
3234 {
3235 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3239 }
3240 let c = &mut self.state.candidates[i];
3241 c.commits = commits;
3242 c.stat = stat;
3243 c.files = files;
3244 let no_changes = commits == 0 || patch.trim().is_empty();
3245 c.empty = no_changes && !interrupted;
3246 if interrupted && no_changes {
3247 c.failed = None;
3248 c.verified_noop = None;
3249 }
3250 if !c.empty {
3251 c.failed = None;
3252 c.verified_noop = None;
3253 }
3254 self.state.event(
3255 "preserve",
3256 format!("candidate {label}: {files} files, {commits} commits"),
3257 );
3258 Ok::<_, anyhow::Error>(())
3259 }
3260 .await;
3261 match result {
3262 Ok(()) => {
3263 self.state.candidate_preservation.remove(&label);
3264 if !interrupted || self.state.candidates[i].files > 0 {
3265 self.state.interrupted_implementations.remove(&label);
3266 }
3267 }
3268 Err(e) => {
3269 let reason = format!("candidate {label} preservation failed: {e:#}");
3270 self.state.event("preserve", &reason);
3271 self.state.candidate_preservation.insert(label, reason);
3272 }
3273 }
3274 if self.state.candidate_preservation.is_empty()
3275 && self.state.status == RunStatus::Blocked
3276 && self.state.candidates.iter().all(|c| c.failed.is_none())
3277 {
3278 self.state.status = RunStatus::Implementing;
3282 self.state.event(
3283 "preserve",
3284 "candidate persistence recovered; implementation can continue",
3285 );
3286 }
3287 self.state.save()?;
3288 }
3289 if !self.state.candidate_preservation.is_empty() {
3290 self.state.status = RunStatus::Blocked;
3291 self.state.save()?;
3292 bail!(
3293 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3294 );
3295 }
3296 Ok(())
3297 }
3298
3299 async fn resume_undelivered(
3327 &mut self,
3328 results: &mut [(usize, SeatState, AgentOutcome)],
3329 sent: &[SeatJob],
3330 prompts: &Prompts,
3331 run_id: &str,
3332 ) {
3333 for (wi, seat, out) in results.iter_mut() {
3334 let Some(dropped) = (match &*out {
3335 AgentOutcome::Dropped(o) => o.dropped.clone(),
3336 _ => None,
3337 }) else {
3338 continue;
3339 };
3340 let Some(job) = sent.get(*wi) else { continue };
3341 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3343 self.state.event(
3344 "implement",
3345 format!(
3346 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3347 work is in the tree",
3348 seat.key, dropped.output_tokens, dropped.why
3349 ),
3350 );
3351 continue;
3352 }
3353 if !has_context(&job.spec, seat, job.sessions) {
3361 self.state.event(
3362 "implement",
3363 format!(
3364 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3365 is no session left to resume",
3366 seat.key, dropped.output_tokens, dropped.why
3367 ),
3368 );
3369 continue;
3370 }
3371 self.state.event(
3372 "implement",
3373 format!(
3374 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3375 conversation",
3376 seat.key, dropped.output_tokens, dropped.why
3377 ),
3378 );
3379 let mut retry = job.clone();
3380 retry.seat = seat.clone();
3381 retry.prompt = prompt::resume_after_drop(&dropped.why);
3382 retry.timeout = retry_budget(job.timeout, true);
3383 retry.stem = format!("{}-resume", job.stem);
3384 let cache = self.state.config.cache_dir();
3385 let ctx = WaveCtx {
3386 carry_seats: false,
3387 pauses: Some((&self.pause, &self.interrupt)),
3388 run: run_id,
3389 node: "implement",
3390 prompts,
3391 cache: cache.as_deref(),
3392 round: None,
3393 };
3394 let (resumed_seat, resumed) =
3395 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3396 *seat = resumed_seat;
3397 *out = resumed;
3398 }
3399 }
3400
3401 async fn resume_seat_handovers(
3466 &mut self,
3467 results: &mut [(usize, SeatState, AgentOutcome)],
3468 sent: &mut [SeatJob],
3469 prompts: &Prompts,
3470 run_id: &str,
3471 ) -> Result<()> {
3472 let instruction = seeded_instruction(&self.state);
3473 let language = self.state.config.graph.language.clone();
3474 let github_language = self.state.config.graph.github_language.clone();
3475 let brief = self
3476 .state
3477 .advice
3478 .as_ref()
3479 .and_then(|a| a.synthesis.as_deref())
3480 .map(str::to_owned);
3481 let attachments = self.state.attachments.clone();
3482 for (wi, seat, out) in results.iter_mut() {
3483 let others: BTreeSet<String> = sent
3486 .iter()
3487 .enumerate()
3488 .filter(|(j, _)| j != wi)
3489 .map(|(_, j)| j.spec.id.clone())
3490 .collect();
3491 let Some(job) = sent.get_mut(*wi) else {
3492 continue;
3493 };
3494 let start = self
3499 .roles
3500 .implementer_roster
3501 .iter()
3502 .position(|s| s.id == job.spec.id)
3503 .unwrap_or(0);
3504 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3505 let mut fallback_attempt = 0usize;
3506 let mut prev: Option<FailClass> = None;
3507 while let Some(cur) = FailClass::of(&*out) {
3508 if !should_hand_over(prev.as_ref(), &cur) {
3509 break;
3510 }
3511 let Some(next) =
3512 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3513 .cloned()
3514 else {
3515 break;
3516 };
3517 tried.insert(next.id.clone());
3518 fallback_attempt += 1;
3519
3520 let coordination = self.coordinate(&job.cwd).await?;
3521 if let Ok(r) = git::rescue_commit(
3522 &job.cwd,
3523 &format!(
3524 "magi: candidate {} (uncommitted work before {} fallback)",
3525 seat.key,
3526 if cur == FailClass::Quota {
3527 "quota"
3528 } else {
3529 "handover"
3530 }
3531 ),
3532 )
3533 .await
3534 {
3535 self.state.note_withheld("implement", &r.withheld);
3536 }
3537 drop(coordination);
3538
3539 record_handover(
3540 &mut self.state,
3541 "implement",
3542 &seat.key,
3543 &seat.agent,
3544 &next.id,
3545 &cur,
3546 &fail_reason(&*out),
3547 );
3548 prev = Some(cur.clone());
3549
3550 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3551 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3552 job.spec = next.clone();
3560 let mut retry = job.clone();
3561 retry.seat = new_seat;
3562 retry.prompt = prompt::implement_in(
3563 &instruction,
3564 &job.cwd.to_string_lossy(),
3565 &language,
3566 &github_language,
3567 brief.as_deref(),
3568 &attachments,
3569 );
3570 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3571 let cache = self.state.config.cache_dir();
3572 let ctx = WaveCtx {
3573 carry_seats: false,
3574 pauses: Some((&self.pause, &self.interrupt)),
3575 run: run_id,
3576 node: "implement",
3577 prompts,
3578 cache: cache.as_deref(),
3579 round: None,
3580 };
3581 let (fallback_seat, fallback_out) = run_one(
3582 retry,
3583 Arc::clone(&self.sem),
3584 &ctx,
3585 &mut self.state,
3586 fallback_attempt,
3587 )
3588 .await;
3589 *seat = fallback_seat;
3590 *out = fallback_out;
3591 }
3592 }
3593 Ok(())
3594 }
3595
3596 async fn resume_unconfirmed_commands(
3620 &mut self,
3621 results: &mut [(usize, SeatState, AgentOutcome)],
3622 sent: &[SeatJob],
3623 prompts: &Prompts,
3624 run_id: &str,
3625 ) {
3626 for (wi, seat, out) in results.iter_mut() {
3627 let AgentOutcome::Ok(o) = &*out else {
3628 continue;
3629 };
3630 if !has_unconfirmed_command(&o.commands) {
3631 continue;
3632 }
3633 let Some(job) = sent.get(*wi) else { continue };
3634 if !has_context(&job.spec, seat, job.sessions) {
3635 self.state.event(
3636 "implement",
3637 format!(
3638 "{}: the reply named a command whose own CLI never confirmed the exit \
3639 status of, but there is no session left to resume",
3640 seat.key
3641 ),
3642 );
3643 continue;
3644 }
3645 self.state.event(
3646 "implement",
3647 format!(
3648 "{}: the reply named a command whose own CLI never confirmed the exit \
3649 status of; resuming the conversation",
3650 seat.key
3651 ),
3652 );
3653 let mut retry = job.clone();
3654 retry.seat = seat.clone();
3655 retry.prompt = prompt::resume_incomplete(
3656 "a command in your last reply had no confirmed exit status",
3657 );
3658 retry.timeout = retry_budget(job.timeout, true);
3659 retry.stem = format!("{}-confirm", job.stem);
3660 let cache = self.state.config.cache_dir();
3661 let ctx = WaveCtx {
3662 carry_seats: false,
3663 pauses: Some((&self.pause, &self.interrupt)),
3664 run: run_id,
3665 node: "implement",
3666 prompts,
3667 cache: cache.as_deref(),
3668 round: None,
3669 };
3670 let (resumed_seat, resumed) =
3671 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3672 *seat = resumed_seat;
3673 *out = resumed;
3674 }
3675 }
3676
3677 async fn continue_fix_report(
3698 &mut self,
3699 mut seat: SeatState,
3700 parse_err: String,
3701 job: &SeatJob,
3702 prompts: &Prompts,
3703 run_id: &str,
3704 round: usize,
3705 ) -> (
3706 SeatState,
3707 Option<FixReport>,
3708 Option<String>,
3709 ContinuationRecord,
3710 ) {
3711 let mut last_err = parse_err;
3712 let mut cumulative_wait_ms = 0u64;
3713 let mut attempts = 0usize;
3714 loop {
3715 if !has_context(&job.spec, &seat, job.sessions) {
3716 self.state.event(
3717 "fix",
3718 format!(
3719 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3720 session left to resume into"
3721 ),
3722 );
3723 let outcome = if attempts == 0 {
3724 ContinuationOutcome::NoSession
3725 } else {
3726 ContinuationOutcome::Exhausted
3727 };
3728 return (
3729 seat,
3730 None,
3731 Some(format!("unparsable fix report: {last_err}")),
3732 ContinuationRecord {
3733 attempts,
3734 cumulative_wait_ms,
3735 outcome,
3736 },
3737 );
3738 }
3739 if attempts >= MAX_FIX_CONTINUATIONS {
3740 self.state.event(
3741 "fix",
3742 format!(
3743 "round {round}: fixer's reply still had no adoption report after \
3744 {attempts} continuation(s) ({last_err}); giving up"
3745 ),
3746 );
3747 return (
3748 seat,
3749 None,
3750 Some(format!(
3751 "unparsable fix report after {attempts} continuation(s): {last_err}"
3752 )),
3753 ContinuationRecord {
3754 attempts,
3755 cumulative_wait_ms,
3756 outcome: ContinuationOutcome::Exhausted,
3757 },
3758 );
3759 }
3760 attempts += 1;
3761 self.state.event(
3762 "fix",
3763 format!(
3764 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3765 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3766 ),
3767 );
3768 let mut retry = job.clone();
3769 retry.seat = seat.clone();
3770 retry.prompt = prompt::resume_incomplete(&last_err);
3771 retry.timeout = retry_budget(job.timeout, true);
3772 retry.stem = format!("{}-continue{attempts}", job.stem);
3773 let cache = self.state.config.cache_dir();
3774 let ctx = WaveCtx {
3775 carry_seats: false,
3776 pauses: Some((&self.pause, &self.interrupt)),
3777 run: run_id,
3778 node: "fix",
3779 prompts,
3780 cache: cache.as_deref(),
3781 round: Some(round),
3782 };
3783 let (resumed_seat, resumed_out) = run_one(
3784 retry,
3785 Arc::clone(&self.sem),
3786 &ctx,
3787 &mut self.state,
3788 attempts,
3789 )
3790 .await;
3791 seat = resumed_seat;
3792 match resumed_out {
3793 AgentOutcome::Ok(o) => {
3794 cumulative_wait_ms += o.duration_ms;
3795 match verdict::extract_json::<FixReport>(&o.text) {
3796 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3797 self.state.event(
3798 "fix",
3799 format!(
3800 "round {round}: fixer's adoption report recovered after \
3801 {attempts} continuation(s)"
3802 ),
3803 );
3804 return (
3805 seat,
3806 Some(report),
3807 None,
3808 ContinuationRecord {
3809 attempts,
3810 cumulative_wait_ms,
3811 outcome: ContinuationOutcome::Resumed,
3812 },
3813 );
3814 }
3815 Ok(_) => {
3823 last_err = "the reply parsed, but it reported a command whose own CLI \
3824 never confirmed an exit status"
3825 .to_owned();
3826 }
3827 Err(e) => last_err = e.to_string(),
3828 }
3829 }
3830 AgentOutcome::Quota(o) => {
3831 cumulative_wait_ms += o.duration_ms;
3832 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3833 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3834 self.state.quota.push(loss);
3835 self.state.event(
3836 "fix",
3837 format!(
3838 "round {round}: continuation rate limited (quota); not retrying now"
3839 ),
3840 );
3841 return (
3842 seat,
3843 None,
3844 Some("rate limited (quota) while recovering the fix report".to_owned()),
3845 ContinuationRecord {
3846 attempts,
3847 cumulative_wait_ms,
3848 outcome: ContinuationOutcome::QuotaLost,
3849 },
3850 );
3851 }
3852 AgentOutcome::Dropped(o) => {
3853 cumulative_wait_ms += o.duration_ms;
3854 let why = o
3855 .dropped
3856 .as_ref()
3857 .map(|d| d.why.as_str())
3858 .unwrap_or("the CLI ended the stream without delivering its answer");
3859 last_err = format!("the CLI dropped the stream ({why})");
3860 }
3861 AgentOutcome::Failed(e) => last_err = e,
3862 AgentOutcome::ResourceBlocked(e) => {
3865 return (
3866 seat,
3867 None,
3868 Some(format!("the continuation was not started: {e}")),
3869 ContinuationRecord {
3870 attempts: attempts.saturating_sub(1),
3871 cumulative_wait_ms,
3872 outcome: ContinuationOutcome::NoSession,
3873 },
3874 );
3875 }
3876 }
3877 }
3878 }
3879
3880 fn after_implement(&mut self) -> Result<()> {
3881 if !self.state.candidate_preservation.is_empty() {
3882 bail!("candidate preservation incomplete; judging is unsafe");
3883 }
3884 if self.state.leaks.is_empty() {
3886 let cfg = self.state.config.blind.clone();
3887 let mut leaks = Vec::new();
3888 for c in &self.state.candidates {
3889 let Some(patch) =
3890 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3891 else {
3892 continue;
3893 };
3894 leaks.extend(blind::scan(
3895 &format!("candidate {} patch", c.label),
3896 &patch,
3897 &cfg.vendor_tokens,
3898 ));
3899 }
3900 if !leaks.is_empty() {
3901 let summary = leaks
3902 .iter()
3903 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3904 .collect::<Vec<_>>()
3905 .join(", ");
3906 match cfg.on_leak {
3907 LeakPolicy::Fail => {
3908 self.state.status = RunStatus::Failed;
3909 self.state
3910 .event("blind", format!("vendor text in a patch: {summary}"));
3911 self.state.leaks = leaks;
3912 self.state.save()?;
3913 self.settle_questions();
3914 bail!(
3915 "blind.on_leak = \"fail\" and vendor text reached a \
3916 judged patch: {summary}"
3917 );
3918 }
3919 LeakPolicy::Redact => self.state.event(
3920 "blind",
3921 format!("redacting vendor text for judging: {summary}"),
3922 ),
3923 LeakPolicy::Warn => self.state.event(
3924 "blind",
3925 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3926 ),
3927 }
3928 self.state.leaks = leaks;
3929 }
3930 }
3931
3932 if self.state.viable().is_empty() {
3933 if self.state.all_candidates_verified_noop() {
3934 self.state.status = RunStatus::VerifiedNoop;
3945 self.state.save()?;
3946 self.settle_questions();
3947 return Ok(());
3948 }
3949 self.state.status = RunStatus::Failed;
3950 self.state.save()?;
3951 self.settle_questions();
3952 bail!("no candidate produced a change; nothing to judge");
3953 }
3954 self.state.status = RunStatus::Judging;
3955 self.state.save()?;
3956 Ok(())
3957 }
3958
3959 async fn judge(&mut self) -> Result<()> {
3962 let run_id = self.state.id.clone();
3967 let prompts = self.state.config.prompts.clone();
3968 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3969 return Ok(());
3970 }
3971 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3972 if viable.len() == 1 {
3973 self.state.judge_skipped = true;
3980 self.state.event(
3981 "judge",
3982 format!(
3983 "only candidate {} produced a change; judging skipped",
3984 viable[0].label
3985 ),
3986 );
3987 self.state.save()?;
3988 return Ok(());
3989 }
3990 self.state.status = RunStatus::Judging;
3991
3992 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3993 let language = self.state.config.graph.language.clone();
3994 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3995 let sessions = self.state.config.graph.sessions;
3996 let artifacts = agent::artifacts_dir(&self.state.dir());
3997 let root = self.state.worktree_root();
3998 let base_short = short(&self.state.base_commit);
3999
4000 let mut jobs = Vec::new();
4001 let mut orders = Vec::new();
4002 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4003 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4004 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4005 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4006 let seat_key = format!("judge-{}", j + 1);
4007 let seat = self.seat(&seat_key, &spec.id);
4008 jobs.push(SeatJob {
4009 prompt: prompt::judge(
4010 &self.state.instruction,
4011 &views,
4012 self.roles.judges.len(),
4013 &base_short,
4014 &language,
4015 ),
4016 spec,
4017 seat,
4018 cwd: root.join(format!("judge-{}", j + 1)),
4019 timeout,
4020 allow_write: false,
4021 sessions,
4022 artifacts: artifacts.clone(),
4023 stem: format!("judge-{}", j + 1),
4024 handover: None,
4025 });
4026 }
4027
4028 self.state.event(
4029 "judge",
4030 format!(
4031 "{} judges ranking {} candidates blind",
4032 jobs.len(),
4033 viable.len()
4034 ),
4035 );
4036 let labels_for_check = labels.clone();
4037 let mut quota_losses = Vec::new();
4038 let cache = self.state.config.cache_dir();
4039 let ctx = WaveCtx {
4040 carry_seats: false,
4041 pauses: Some((&self.pause, &self.interrupt)),
4042 run: &run_id,
4043 node: "judge",
4044 prompts: &prompts,
4045 cache: cache.as_deref(),
4046 round: None,
4047 };
4048 let results = ask_json_wave::<Ranking>(
4049 jobs,
4050 Arc::clone(&self.sem),
4051 self.state.config.graph.retries,
4052 &self.roles.judge_roster,
4053 &ctx,
4054 &mut quota_losses,
4055 &mut self.state,
4056 &move |r: &Ranking| r.validate(&labels_for_check),
4057 )
4058 .await;
4059 if self.state.resource_block.is_some() {
4060 return self.stop_for_resource();
4061 }
4062 self.state.quota.extend(quota_losses);
4063
4064 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4065 let agent_id = seat.agent.clone();
4066 self.state.seats.insert(seat.key.clone(), seat);
4067 let mut record = Judgement {
4068 judge: j + 1,
4069 seat: format!("judge-{}", j + 1),
4070 agent: agent_id,
4071 ranking: Vec::new(),
4072 reasons: BTreeMap::new(),
4073 confidence: None,
4074 strengths: BTreeMap::new(),
4075 order: orders[j].clone(),
4076 failed: None,
4077 duration_ms: 0,
4078 };
4079 match res {
4080 Ok((ranking, out)) => {
4081 record.ranking = ranking.normalized();
4082 record.reasons = ranking.reasons;
4083 record.strengths = ranking.strengths;
4084 record.confidence = ranking.confidence;
4085 record.duration_ms = out.duration_ms;
4086 self.state.event(
4087 "judge",
4088 format!(
4089 "judge {} ranked {}",
4090 j + 1,
4091 record.ranking.iter().collect::<String>()
4092 ),
4093 );
4094 }
4095 Err(e) => {
4096 record.failed = Some(e.to_string());
4097 self.state
4098 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4099 }
4100 }
4101 self.state.judgements.push(record);
4102 self.state.save()?;
4103 }
4104 Ok(())
4105 }
4106
4107 async fn deliberate(&mut self) -> Result<()> {
4110 if self.hold_base_sync().await? {
4111 return Ok(());
4112 }
4113 let run_id = self.state.id.clone();
4118 let prompts = self.state.config.prompts.clone();
4119 let rounds = self.state.config.graph.deliberate_rounds;
4120 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4122 return Ok(());
4123 }
4124 let tops: Vec<char> = self
4125 .state
4126 .judgements
4127 .iter()
4128 .filter_map(|j| j.ranking.first().copied())
4129 .collect();
4130 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4131 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4132 self.state.event(
4133 "deliberate",
4134 format!("judges agreed on {} outright; no deliberation", tops[0]),
4135 );
4136 }
4137 self.state.status = RunStatus::Voting;
4138 self.state.save()?;
4139 return Ok(());
4140 }
4141
4142 self.state.status = RunStatus::Deliberating;
4143 self.state.event(
4144 "deliberate",
4145 format!(
4146 "split: first choices were {} — opening {rounds} round(s)",
4147 tops.iter().collect::<String>()
4148 ),
4149 );
4150
4151 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4152 let language = self.state.config.graph.language.clone();
4153 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4154 let sessions = self.state.config.graph.sessions;
4155 let artifacts = agent::artifacts_dir(&self.state.dir());
4156 let root = self.state.worktree_root();
4157 let base_short = short(&self.state.base_commit);
4158
4159 for round in self.state.deliberation.len() + 1..=rounds {
4163 let mut turns: Vec<DeliberationTurn> = Vec::new();
4164 let mut round_losses = Vec::new();
4165 let mut asked = 0;
4166 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4167 if self.state.judgements[j].failed.is_some() {
4168 continue;
4169 }
4170 let seat_key = format!("judge-{}", j + 1);
4171 let spec = self.occupant(&seat_key, spec);
4172 let mut seat = self.seat(&seat_key, &spec.id);
4173 let transcript = self.transcript(&turns, j);
4174 let build = |context: Option<&str>| {
4175 prompt::deliberate(
4176 &self.state.instruction,
4177 context,
4178 &transcript,
4179 round,
4180 rounds,
4181 &language,
4182 )
4183 };
4184 let block = self.candidate_block(&viable, &base_short);
4185 let full = build(Some(&block));
4186 let text = if has_context(&spec, &seat, sessions) {
4187 build(None)
4188 } else {
4189 full.clone()
4190 };
4191 let job = SeatJob {
4192 spec,
4193 seat: seat.clone(),
4194 prompt: text,
4195 cwd: root.join(format!("judge-{}", j + 1)),
4196 timeout,
4197 allow_write: false,
4198 sessions,
4199 artifacts: artifacts.clone(),
4200 stem: format!("delib-{round}-judge-{}", j + 1),
4201 handover: Some(full),
4202 };
4203 let cache = self.state.config.cache_dir();
4204 let ctx = WaveCtx {
4205 carry_seats: false,
4206 pauses: Some((&self.pause, &self.interrupt)),
4207 run: &run_id,
4208 node: "deliberate",
4209 prompts: &prompts,
4210 cache: cache.as_deref(),
4211 round: None,
4212 };
4213 let mut losses = Vec::new();
4217 asked += 1;
4218 let mut results = ask_wave_with::<String>(
4219 vec![job],
4220 Arc::clone(&self.sem),
4221 0,
4222 &self.roles.judge_roster,
4223 &ctx,
4224 &mut losses,
4225 &mut self.state,
4226 &|text: &str| {
4227 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4228 },
4229 )
4230 .await;
4231 if self.state.resource_block.is_some() {
4232 return self.stop_for_resource();
4233 }
4234 round_losses.extend(losses.iter().cloned());
4235 self.state.quota.extend(losses);
4236 let (updated, res, _) = results.pop().expect("one job in, one result out");
4237 seat = updated;
4238 let agent_id = seat.agent.clone();
4239 self.state.seats.insert(seat.key.clone(), seat);
4240 let body = match res {
4241 Ok((body, _)) => body,
4242 Err(e) => {
4245 self.state
4246 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4247 continue;
4248 }
4249 };
4250 let tentative = verdict::extract_json::<Position>(&body)
4251 .ok()
4252 .and_then(|p| p.tentative)
4253 .and_then(|s| s.trim().chars().next())
4254 .map(|c| c.to_ascii_uppercase());
4255 self.state.event(
4256 "deliberate",
4257 format!(
4258 "round {round}: judge {} now favours {}",
4259 j + 1,
4260 tentative.map_or("—".to_owned(), |c| c.to_string())
4261 ),
4262 );
4263 turns.push(DeliberationTurn {
4264 judge: j + 1,
4265 agent: agent_id,
4266 body: blind::sanitize_prose(&body, &self.state.config.blind),
4267 tentative,
4268 });
4269 }
4270 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4271 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4272 return self.stop_for_resource();
4273 }
4274 self.state
4275 .deliberation
4276 .push(DeliberationRound { round, turns });
4277 self.state.save()?;
4278 }
4279
4280 self.state.status = RunStatus::Voting;
4281 self.state.save()?;
4282 Ok(())
4283 }
4284
4285 async fn vote(&mut self) -> Result<()> {
4288 if self.hold_base_sync().await? {
4289 return Ok(());
4290 }
4291 let run_id = self.state.id.clone();
4296 let prompts = self.state.config.prompts.clone();
4297 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4298 return Ok(());
4299 }
4300 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4301 if viable.len() == 1 {
4302 return Ok(());
4303 }
4304 self.state.status = RunStatus::Voting;
4305
4306 let language = self.state.config.graph.language.clone();
4307 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4308 let sessions = self.state.config.graph.sessions;
4309 let artifacts = agent::artifacts_dir(&self.state.dir());
4310 let root = self.state.worktree_root();
4311 let base_short = short(&self.state.base_commit);
4312 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4313
4314 let mut jobs = Vec::new();
4315 let mut seats_at = Vec::new();
4316 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4317 if self
4318 .state
4319 .judgements
4320 .get(j)
4321 .is_some_and(|r| r.failed.is_some())
4322 {
4323 continue;
4324 }
4325 let seat_key = format!("judge-{}", j + 1);
4326 let spec = self.occupant(&seat_key, spec);
4327 let seat = self.seat(&seat_key, &spec.id);
4328 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4329 let text = if has_context(&spec, &seat, sessions) {
4330 prompt::final_vote(&viable, &language)
4331 } else {
4332 full.clone()
4333 };
4334 jobs.push(SeatJob {
4335 spec,
4336 seat,
4337 prompt: text,
4338 cwd: root.join(format!("judge-{}", j + 1)),
4339 timeout,
4340 allow_write: false,
4341 sessions,
4342 artifacts: artifacts.clone(),
4343 stem: format!("vote-judge-{}", j + 1),
4344 handover: Some(full),
4345 });
4346 seats_at.push(j);
4347 }
4348
4349 self.state.event(
4350 "vote",
4351 format!(
4352 "collecting {} final votes one by one, privately",
4353 jobs.len()
4354 ),
4355 );
4356 let allowed = viable.clone();
4357 let mut quota_losses = Vec::new();
4358 let cache = self.state.config.cache_dir();
4359 let ctx = WaveCtx {
4360 carry_seats: false,
4361 pauses: Some((&self.pause, &self.interrupt)),
4362 run: &run_id,
4363 node: "vote",
4364 prompts: &prompts,
4365 cache: cache.as_deref(),
4366 round: None,
4367 };
4368 let results = ask_json_wave::<FinalVote>(
4369 jobs,
4370 Arc::clone(&self.sem),
4371 self.state.config.graph.retries,
4372 &self.roles.judge_roster,
4373 &ctx,
4374 &mut quota_losses,
4375 &mut self.state,
4376 &move |v: &FinalVote| match v.label() {
4377 Some(c) if allowed.contains(&c) => Ok(()),
4378 other => bail!("vote {other:?} is not one of {allowed:?}"),
4379 },
4380 )
4381 .await;
4382 if self.state.resource_block.is_some() {
4383 return self.stop_for_resource();
4384 }
4385 self.state.quota.extend(quota_losses);
4386
4387 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4388 let agent_id = seat.agent.clone();
4389 self.state.seats.insert(seat.key.clone(), seat);
4390 let initial = self
4391 .state
4392 .judgements
4393 .get(j)
4394 .and_then(|r| r.ranking.first().copied());
4395 let mut record = VoteRecord {
4396 judge: j + 1,
4397 agent: agent_id,
4398 vote: None,
4399 reason: String::new(),
4400 changed: false,
4401 };
4402 match res {
4403 Ok((v, _)) => {
4404 record.vote = v.label();
4405 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4406 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4407 self.state.event(
4408 "vote",
4409 format!(
4410 "judge {} voted {}{}",
4411 j + 1,
4412 record.vote.unwrap_or('?'),
4413 if record.changed { " (changed)" } else { "" }
4414 ),
4415 );
4416 }
4417 Err(e) => {
4418 self.state
4419 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4420 }
4421 }
4422 self.state.votes.push(record);
4423 self.state.save()?;
4424 }
4425 Ok(())
4426 }
4427
4428 fn tally(&mut self) -> Result<()> {
4431 if self.state.tally.is_some() {
4432 return Ok(());
4433 }
4434 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4435 let tops: Vec<char> = self
4436 .state
4437 .judgements
4438 .iter()
4439 .filter_map(|j| j.ranking.first().copied())
4440 .collect();
4441 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4442
4443 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4446 let mut cast: Vec<char> = Vec::new();
4447 for (i, j) in self.state.judgements.iter().enumerate() {
4448 let vote = self
4449 .state
4450 .votes
4451 .iter()
4452 .find(|v| v.judge == i + 1)
4453 .and_then(|v| v.vote)
4454 .or_else(|| j.ranking.first().copied());
4455 if let Some(v) = vote {
4456 *first_choice.entry(v).or_insert(0) += 1;
4457 cast.push(v);
4458 }
4459 }
4460
4461 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4462 for j in &self.state.judgements {
4463 let n = j.ranking.len();
4464 for (pos, label) in j.ranking.iter().enumerate() {
4465 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4466 }
4467 }
4468
4469 let best = first_choice.values().copied().max().unwrap_or(0);
4470 let mut leaders: Vec<char> = first_choice
4471 .iter()
4472 .filter(|(_, v)| **v == best)
4473 .map(|(k, _)| *k)
4474 .collect();
4475 let mut tie_break = None;
4476 if leaders.len() > 1 {
4477 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4478 let borda_leaders: Vec<char> = leaders
4479 .iter()
4480 .copied()
4481 .filter(|l| borda[l] == top_borda)
4482 .collect();
4483 tie_break = Some(if borda_leaders.len() == 1 {
4484 format!(
4485 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4486 leaders.len()
4487 )
4488 } else {
4489 format!(
4490 "{} way tie on both first-choice votes and Borda points, broken by label order",
4491 leaders.len()
4492 )
4493 });
4494 leaders = borda_leaders;
4495 leaders.sort_unstable();
4496 }
4497 let winner = *leaders
4498 .first()
4499 .or(viable.first())
4500 .context("no candidate to declare a winner from")?;
4501
4502 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4503 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4504 let deliberated = !self.state.deliberation.is_empty();
4505
4506 let quota_seats: std::collections::BTreeSet<&str> =
4510 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4511 let mut present = 0usize;
4512 for (i, j) in self.state.judgements.iter().enumerate() {
4513 if quota_seats.contains(j.seat.as_str()) {
4514 continue;
4515 }
4516 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4517 let voted = self
4518 .state
4519 .votes
4520 .iter()
4521 .any(|v| v.judge == i + 1 && v.vote.is_some());
4522 if ranked || voted {
4523 present += 1;
4524 }
4525 }
4526 let needs_quorum = viable.len() > 1;
4532 let judges_total = if needs_quorum {
4533 self.roles.judges.len()
4534 } else {
4535 0
4536 };
4537 let quorum = if needs_quorum {
4538 judges_total / 2 + 1
4539 } else {
4540 0
4541 };
4542 let met_quorum = !needs_quorum || present >= quorum;
4543 let uncontested = (!needs_quorum).then(|| {
4544 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4545 });
4546
4547 self.state.event(
4548 "tally",
4549 match &uncontested {
4550 Some(reason) => format!("winner {winner} — {reason}"),
4551 None => format!(
4552 "winner {winner} — votes {} | initial {} | {} changed | \
4553 {present}/{judges_total} judges{}",
4554 first_choice
4555 .iter()
4556 .map(|(k, v)| format!("{k}:{v}"))
4557 .collect::<Vec<_>>()
4558 .join(" "),
4559 if unanimous_initial {
4560 "unanimous"
4561 } else {
4562 "split"
4563 },
4564 changed_votes,
4565 if met_quorum {
4566 String::new()
4567 } else {
4568 format!(" — below quorum ({quorum} required)")
4569 },
4570 ),
4571 },
4572 );
4573 if !met_quorum {
4574 self.state.event(
4575 "stall",
4576 format!(
4577 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4578 the run stops here, resumable"
4579 ),
4580 );
4581 }
4582 self.state.tally = Some(Tally {
4583 first_choice,
4584 borda,
4585 winner,
4586 rankings: tops.len(),
4587 unanimous_initial,
4588 deliberated,
4589 changed_votes,
4590 unanimous_final,
4591 tie_break,
4592 judges: judges_total,
4593 present,
4594 quorum,
4595 met_quorum,
4596 uncontested,
4597 });
4598 self.state.status = if met_quorum {
4599 RunStatus::Reviewing
4600 } else {
4601 RunStatus::Stalled
4602 };
4603 self.state.save()?;
4604 Ok(())
4605 }
4606
4607 #[allow(clippy::too_many_lines)]
4628 async fn recover_stall(&mut self) -> Result<bool> {
4629 let run_id = self.state.id.clone();
4634 let prompts = self.state.config.prompts.clone();
4635 let quota_seats: BTreeSet<&str> =
4640 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4641 let absent: Vec<String> = self
4642 .state
4643 .judgements
4644 .iter()
4645 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4646 .map(|j| j.seat.clone())
4647 .collect();
4648 if absent.is_empty() {
4649 return Ok(false);
4650 }
4651 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4652 if viable.len() <= 1 {
4653 return Ok(false);
4654 }
4655 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4656 let language = self.state.config.graph.language.clone();
4657 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4658 let sessions = self.state.config.graph.sessions;
4659 let artifacts = agent::artifacts_dir(&self.state.dir());
4660 let root = self.state.worktree_root();
4661 let base_short = short(&self.state.base_commit);
4662 let candidates: Vec<Candidate> = viable.clone();
4663
4664 let mut positions: Vec<usize> = absent
4666 .iter()
4667 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4668 .collect();
4669 if positions.is_empty() {
4670 return Ok(false);
4671 }
4672 positions.sort_unstable();
4673 positions.dedup();
4674
4675 let mut judge_jobs = Vec::new();
4677 for &j in &positions {
4678 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4679 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4680 let seat_key = format!("judge-{}", j + 1);
4681 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4682 let seat = self.seat(&seat_key, &spec.id);
4683 judge_jobs.push(SeatJob {
4684 spec,
4685 seat,
4686 prompt: prompt::judge(
4687 &self.state.instruction,
4688 &views,
4689 self.roles.judges.len(),
4690 &base_short,
4691 &language,
4692 ),
4693 cwd: root.join(seat_key),
4694 timeout,
4695 allow_write: false,
4696 sessions,
4697 artifacts: artifacts.clone(),
4698 stem: format!("judge-{}-recover", j + 1),
4699 handover: None,
4700 });
4701 }
4702
4703 let labels_for_check = labels.clone();
4704 let mut judge_losses = Vec::new();
4705 let retries = self.state.config.graph.retries;
4706 let cache = self.state.config.cache_dir();
4707 let ctx = WaveCtx {
4708 carry_seats: false,
4709 pauses: Some((&self.pause, &self.interrupt)),
4710 run: &run_id,
4711 node: "judge",
4712 prompts: &prompts,
4713 cache: cache.as_deref(),
4714 round: None,
4715 };
4716 let results = ask_json_wave::<Ranking>(
4717 judge_jobs,
4718 Arc::clone(&self.sem),
4719 retries,
4720 &self.roles.judge_roster,
4721 &ctx,
4722 &mut judge_losses,
4723 &mut self.state,
4724 &move |r: &Ranking| r.validate(&labels_for_check),
4725 )
4726 .await;
4727 if self.state.resource_block.is_some() {
4728 self.stop_for_resource()?;
4729 return Ok(false);
4730 }
4731
4732 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4734 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4735 let agent_id = seat.agent.clone();
4736 self.state.seats.insert(seat.key.clone(), seat);
4737 let record = &mut self.state.judgements[j];
4738 match res {
4739 Ok((ranking, out)) => {
4740 record.agent = agent_id;
4741 record.ranking = ranking.normalized();
4742 record.reasons = ranking.reasons;
4743 record.strengths = ranking.strengths;
4744 record.confidence = ranking.confidence;
4745 record.failed = None;
4746 record.duration_ms = out.duration_ms;
4747 recovered.insert(j);
4748 self.state.event(
4749 "recover",
4750 format!("judge {} ranked again after the limit", j + 1),
4751 );
4752 }
4753 Err(e) => {
4754 self.state
4755 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4756 }
4757 }
4758 }
4759
4760 let mut vote_jobs = Vec::new();
4762 let mut vote_pos: Vec<usize> = Vec::new();
4763 for &j in &recovered {
4764 let seat_key = format!("judge-{}", j + 1);
4765 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4766 let seat = self.seat(&seat_key, &spec.id);
4767 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4768 let text = if has_context(&spec, &seat, sessions) {
4769 prompt::final_vote(&labels, &language)
4770 } else {
4771 full.clone()
4772 };
4773 vote_jobs.push(SeatJob {
4774 spec,
4775 seat,
4776 prompt: text,
4777 cwd: root.join(seat_key),
4778 timeout,
4779 allow_write: false,
4780 sessions,
4781 artifacts: artifacts.clone(),
4782 stem: format!("vote-judge-{}-recover", j + 1),
4783 handover: Some(full),
4784 });
4785 vote_pos.push(j);
4786 }
4787 let allowed = labels.clone();
4788 let mut vote_losses = Vec::new();
4789 let vote_retries = self.state.config.graph.retries;
4790 let vote_cache = self.state.config.cache_dir();
4791 let ctx = WaveCtx {
4792 carry_seats: false,
4793 pauses: Some((&self.pause, &self.interrupt)),
4794 run: &run_id,
4795 node: "vote",
4796 prompts: &prompts,
4797 cache: vote_cache.as_deref(),
4798 round: None,
4799 };
4800 let votes = ask_json_wave::<FinalVote>(
4801 vote_jobs,
4802 Arc::clone(&self.sem),
4803 vote_retries,
4804 &self.roles.judge_roster,
4805 &ctx,
4806 &mut vote_losses,
4807 &mut self.state,
4808 &move |v: &FinalVote| match v.label() {
4809 Some(c) if allowed.contains(&c) => Ok(()),
4810 other => bail!("vote {other:?} is not one of {allowed:?}"),
4811 },
4812 )
4813 .await;
4814 if self.state.resource_block.is_some() {
4815 self.stop_for_resource()?;
4816 return Ok(false);
4817 }
4818 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4819 let agent_id = seat.agent.clone();
4820 self.state.seats.insert(seat.key.clone(), seat);
4821 match res {
4822 Ok((v, _)) => {
4823 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4824 rec.vote = v.label();
4825 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4826 } else {
4827 self.state.votes.push(VoteRecord {
4828 judge: j + 1,
4829 agent: agent_id,
4830 vote: v.label(),
4831 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4832 changed: false,
4833 });
4834 }
4835 self.state.event(
4836 "recover",
4837 format!("judge {} voted again after the limit", j + 1),
4838 );
4839 }
4840 Err(e) => {
4841 self.state
4842 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4843 }
4844 }
4845 }
4846
4847 let recovered_keys: BTreeSet<String> = recovered
4851 .iter()
4852 .map(|&j| format!("judge-{}", j + 1))
4853 .collect();
4854 self.state
4855 .quota
4856 .retain(|q| !recovered_keys.contains(&q.seat));
4857 for loss in judge_losses.into_iter().chain(vote_losses) {
4861 if recovered_keys.contains(&loss.seat) {
4862 continue;
4863 }
4864 self.state.quota.retain(|q| q.seat != loss.seat);
4865 self.state.quota.push(loss);
4866 }
4867
4868 self.state.tally = None;
4870 self.tally()?;
4871 Ok(self
4872 .state
4873 .tally
4874 .as_ref()
4875 .map(|t| t.met_quorum)
4876 .unwrap_or(false))
4877 }
4878
4879 async fn graft(&mut self) -> Result<()> {
4895 let Err(e) = self.graft_inner().await else {
4896 return Ok(());
4897 };
4898 if e.is::<crate::repo_lock::ResourceBlocked>() {
4899 return Err(e);
4900 }
4901 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4902 self.state.status = RunStatus::Blocked;
4903 self.state.event("graft", format!("blocked: {e:#}"));
4904 self.state.save()?;
4905 return Err(e);
4906 }
4907 if let Some(rec) = self.state.graft.clone()
4912 && !rec.outcome.settled()
4913 {
4914 if let Some(winner) = self.state.winner().cloned() {
4915 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4916 if re.is::<crate::repo_lock::ResourceBlocked>() {
4917 return Err(re);
4918 }
4919 self.state.status = RunStatus::Blocked;
4920 self.state.event("graft", format!("blocked: {re:#}"));
4921 self.state.save()?;
4922 return Err(re);
4923 }
4924 }
4925 if let Some(g) = self.state.graft.as_mut() {
4926 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4927 }
4928 }
4929 self.state.event("graft", format!("skipped: {e:#}"));
4930 if let Err(e) = self.state.save() {
4931 tracing::warn!("save after graft: {e:#}");
4932 }
4933 Ok(())
4934 }
4935
4936 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4939 let _coordination = self.coordinate(wt).await?;
4940 crate::recovery::restore_repair(wt, pre)
4941 .await
4942 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4943 }
4944
4945 fn graft_skipped(&mut self, why: &str) {
4947 let message = format!("skipped: {why}");
4948 if !self
4949 .state
4950 .events
4951 .iter()
4952 .any(|e| e.node == "graft" && e.message == message)
4953 {
4954 self.state.event("graft", message);
4955 }
4956 }
4957
4958 async fn graft_inner(&mut self) -> Result<()> {
4959 if let Some(rec) = self.state.graft.clone()
4962 && !rec.outcome.settled()
4963 {
4964 if self.state.reviews.is_empty()
4968 && let Some(winner) = self.state.winner().cloned()
4969 {
4970 self.graft_restore(&winner.worktree, &rec.pre).await?;
4971 }
4972 if rec.outcome == GraftOutcome::WaitingForRestore {
4973 self.state.graft = None;
4974 self.state.save()?;
4975 } else {
4976 if let Some(g) = self.state.graft.as_mut() {
4977 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4978 }
4979 self.state.event(
4980 "graft",
4981 "rolled back: the previous round was interrupted; the winner is untouched",
4982 );
4983 self.state.save()?;
4984 return Ok(());
4985 }
4986 }
4987 let plan = match graft::plan(&self.state) {
4988 Ok(plan) => plan,
4989 Err(skip) => {
4990 if skip != graft::Skip::AlreadyDone {
4991 self.graft_skipped(skip.reason());
4992 self.state.save()?;
4993 }
4994 return Ok(());
4995 }
4996 };
4997 let Some(winner) = self.state.winner().cloned() else {
4998 return Ok(());
4999 };
5000 let wt = winner.worktree.clone();
5001 if !git::is_clean(&wt).await? {
5002 self.graft_skipped("the winner's worktree has uncommitted changes");
5003 self.state.save()?;
5004 return Ok(());
5005 }
5006 let pre = git::rev_parse(&wt, "HEAD").await?;
5007 self.state.graft = Some(GraftRecord {
5010 pre: pre.clone(),
5011 post: None,
5012 lines: None,
5013 limit: None,
5014 agent: None,
5015 outcome: GraftOutcome::Running,
5016 requested: graft::requested(&plan),
5017 adoption: None,
5018 adoption_error: None,
5019 });
5020 self.state.event(
5021 "graft",
5022 format!(
5023 "porting strengths of candidate(s) {} onto {}",
5024 plan.strengths
5025 .iter()
5026 .map(|(l, _, _)| l.to_string())
5027 .collect::<Vec<_>>()
5028 .join(", "),
5029 plan.winner
5030 ),
5031 );
5032 self.state.save()?;
5033
5034 let fix_prompt = prompt::graft(
5035 &self.state.instruction,
5036 plan.winner,
5037 &plan.strengths,
5038 &self.state.config.graph.language,
5039 );
5040 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5041 let sessions = self.state.config.graph.sessions;
5042 let artifacts = agent::artifacts_dir(&self.state.dir());
5043 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5044 let (_, seat, out) = self
5045 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5046 SeatJob {
5047 prompt: fix_prompt.clone(),
5048 spec,
5049 seat,
5050 cwd: wt.clone(),
5051 timeout,
5052 allow_write: true,
5053 sessions,
5054 artifacts: artifacts.clone(),
5055 stem: "graft".to_owned(),
5056 handover: None,
5057 }
5058 })
5059 .await;
5060 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5061 self.state.graft.as_mut().expect("graft record").outcome =
5064 GraftOutcome::WaitingForRestore;
5065 return self.stop_for_resource();
5066 }
5067 let mut why: Option<String> = None;
5068 match &out {
5069 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5070 why = Some("the fixer produced nothing usable".to_owned());
5071 }
5072 AgentOutcome::Ok(_) => {}
5073 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5074 AgentOutcome::Quota(o) => {
5075 self.state.quota.push(QuotaLoss::from_report(
5076 seat.key.clone(),
5077 "graft",
5078 o.quota.as_ref(),
5079 ));
5080 why = Some("rate limited (quota)".to_owned());
5081 }
5082 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5083 why = Some(format!(
5084 "the fixer failed: {}",
5085 e.lines()
5086 .next()
5087 .unwrap_or("")
5088 .chars()
5089 .take(160)
5090 .collect::<String>()
5091 ));
5092 }
5093 }
5094 let agent_id = seat.agent.clone();
5095 self.state.seats.insert(seat.key.clone(), seat);
5096
5097 let mut post = None;
5098 let mut lines = None;
5099 let _coordination = self.coordinate(&wt).await?;
5100 if why.is_none() {
5101 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5102 if let Ok(r) = &rescued {
5103 self.state.note_withheld("graft", &r.withheld);
5104 }
5105 let head = git::rev_parse(&wt, "HEAD").await?;
5106 let dirty = match &rescued {
5110 Ok(r) => git::status_porcelain(&wt)
5111 .await
5112 .map(|st| {
5113 st.lines().any(|l| {
5114 !r.withheld
5115 .iter()
5116 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5117 })
5118 })
5119 .unwrap_or(true),
5120 Err(_) => true,
5121 };
5122 if dirty {
5123 why = Some("the fixer left changes that could not be committed".to_owned());
5124 } else if head == pre {
5125 why = Some("the fixer changed nothing".to_owned());
5126 } else {
5127 lines = git::changed_lines(&wt, &pre, &head).await;
5128 post = Some(head);
5129 }
5130 }
5131 let (adoption, adoption_error) = match &out {
5132 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5133 _ => (None, None),
5134 };
5135 let requested = graft::requested(&plan);
5136 let outcome = match why {
5137 None => {
5138 self.state.event(
5139 "graft",
5140 format!(
5141 "applied: {}",
5142 lines.map_or_else(
5143 || "changed lines unavailable".to_owned(),
5144 |n| format!("{n} changed lines")
5145 )
5146 ),
5147 );
5148 GraftOutcome::Applied
5149 }
5150 Some(why) => {
5151 crate::recovery::restore_repair(&wt, &pre)
5154 .await
5155 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5156 self.state.event(
5157 "graft",
5158 format!("rolled back: {why}; the winner is untouched"),
5159 );
5160 GraftOutcome::RolledBack(why)
5161 }
5162 };
5163 self.state.graft = Some(GraftRecord {
5164 pre,
5165 post,
5166 lines,
5167 limit: None,
5168 agent: Some(agent_id),
5169 outcome,
5170 requested,
5171 adoption,
5172 adoption_error,
5173 });
5174 self.state.save()?;
5175 Ok(())
5176 }
5177
5178 async fn fold_losers(&mut self) -> Result<()> {
5181 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5182 return Ok(());
5183 };
5184 if !self.state.candidate_preservation.is_empty() {
5185 bail!("candidate preservation incomplete; refusing cleanup");
5186 }
5187 let repo = self.state.repo.clone();
5188 let _coordination = self.coordinate(&repo).await?;
5189 let mut folded = Vec::new();
5190 for i in 0..self.state.candidates.len() {
5191 let c = &self.state.candidates[i];
5192 if c.label == winner || c.folded {
5193 continue;
5194 }
5195 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5196 remove_preserved_worktree(&repo, &wt).await?;
5197 if git::branch_exists(&repo, &branch).await?
5198 && !git::branch_delete(&repo, &branch).await?
5199 {
5200 bail!("candidate branch removal refused");
5201 }
5202 self.state.candidates[i].folded = true;
5203 self.state.save()?;
5204 folded.push(label.to_string());
5205 }
5206 let root = self.state.worktree_root();
5208 for j in 1..=self.roles.judges.len() {
5209 let wt = root.join(format!("judge-{j}"));
5210 if wt.exists() {
5211 remove_preserved_worktree(&repo, &wt).await?;
5212 }
5213 }
5214 if self.state.config.graph.advise {
5217 for k in 1..=self.state.config.graph.advisors {
5218 let wt = root.join(format!("advisor-{k}"));
5219 if wt.exists() {
5220 remove_preserved_worktree(&repo, &wt).await?;
5221 }
5222 }
5223 }
5224 if !folded.is_empty() {
5225 self.state
5226 .event("fold", format!("folded candidates {}", folded.join(", ")));
5227 self.state.save()?;
5228 }
5229 Ok(())
5230 }
5231
5232 async fn hold_base_sync(&mut self) -> Result<bool> {
5235 self.hold_base_sync_as(None).await
5236 }
5237
5238 async fn hold_base_sync_kind(
5239 &mut self,
5240 kind: crate::base_sync_recovery::StopKind,
5241 ) -> Result<()> {
5242 self.hold_base_sync_as(Some(kind)).await?;
5243 Ok(())
5244 }
5245
5246 async fn hold_base_sync_as(
5247 &mut self,
5248 kind: Option<crate::base_sync_recovery::StopKind>,
5249 ) -> Result<bool> {
5250 if !self.state.base_sync_blocked() {
5251 self.state.base_sync_recovery.stop = None;
5252 return Ok(false);
5253 }
5254 let head = match self.state.winner() {
5255 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5256 .await
5257 .unwrap_or_default(),
5258 None => String::new(),
5259 };
5260 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5261 && !authorization.consumed
5262 && authorization.stop.head != head
5263 {
5264 authorization.refuse(format!(
5265 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5266 authorization.stop.head
5267 ));
5268 }
5269 self.state.record_base_sync_stop(&head);
5270 if let Some(kind) = kind {
5271 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5272 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5273 stop.kind = kind;
5274 }
5275 }
5276 self.state.save()?;
5277 Ok(true)
5278 }
5279
5280 async fn sync_to_base(&mut self) -> Result<()> {
5322 if self.state.status == RunStatus::AlreadyInBase {
5323 return Ok(());
5324 }
5325 self.hold_base_sync().await?;
5326 let conflicted = self
5327 .state
5328 .base_sync
5329 .as_ref()
5330 .is_some_and(|s| s.conflict.is_some());
5331 let Some(winner) = self.state.winner().cloned() else {
5332 return Ok(());
5333 };
5334
5335 let repo = self.state.repo.clone();
5336 let remote = self.state.config.merge.remote.clone();
5337 let base_branch = self.state.base_branch.clone();
5338 let tracking = format!("{remote}/{base_branch}");
5339
5340 let fetched_base =
5341 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5342 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5347 return Ok(());
5348 };
5349
5350 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5351 self.state.ensure_base_episode();
5352 let mut pending = self
5353 .state
5354 .base_sync_recovery
5355 .operations
5356 .last()
5357 .filter(|o| !o.finished)
5358 .cloned();
5359 let scratch = self.state.dir().join("base-sync");
5360 if git::rebase_in_progress(&scratch).await {
5361 if pending.is_none()
5364 && self.state.base_sync_recovery.operations.is_empty()
5365 && !conflicted
5366 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5367 .await
5368 .ok()
5369 .as_deref()
5370 == Some(head.as_str())
5371 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5372 && git::is_clean(&winner.worktree).await?
5373 {
5374 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5375 if self.state.base_sync.is_none() {
5376 self.state.base_sync = Some(BaseSync {
5377 tip: saved_tip.clone(),
5378 behind: 0,
5379 attempts: 0,
5380 conflict: None,
5381 already_in: None,
5382 });
5383 }
5384 self.state.reserve_base_operation(&head, &saved_tip);
5385 self.state.event(
5386 "base-recovery",
5387 "imported standing legacy rebase as a spent synchronization reservation",
5388 );
5389 self.state.save()?;
5390 pending = self.state.base_sync_recovery.operations.last().cloned();
5391 }
5392 let expected = pending.as_ref().map(|op| op.tip.as_str());
5393 if expected.is_none()
5394 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5395 {
5396 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5397 if self.state.base_sync.is_none() {
5398 self.state.base_sync = Some(BaseSync {
5399 tip: tip.clone(),
5400 behind: 0,
5401 attempts: 0,
5402 conflict: None,
5403 already_in: None,
5404 });
5405 }
5406 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5407 self.state.event("land", why);
5408 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5409 .await?;
5410 return Ok(());
5411 }
5412 }
5413 if let Some(op) = &pending {
5414 let mut replay_proven =
5416 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5417 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5418 let wait = repository_wait(&self.state, None);
5419 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5420 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5421 Ok(true) => {
5422 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5423 self.state.save()?;
5424 git::worktree_remove(&repo, &scratch).await.ok();
5425 replay_proven = true;
5426 }
5427 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5428 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5429 }
5430 }
5431 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5432 if self.state.foreign_checkout.is_some() {
5433 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5434 .await
5435 .ok()
5436 .as_deref()
5437 == Some(op.head.as_str())
5438 && git::rev_parse(&repo, &format!("{head}^2"))
5439 .await
5440 .ok()
5441 .as_deref()
5442 == Some(op.tip.as_str());
5443 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5444 .await
5445 .is_ok_and(|m| m.is_empty())
5446 && let Ok(added) =
5447 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5448 {
5449 replay_proven = true;
5450 for commit in added {
5451 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5452 replay_proven = false;
5453 break;
5454 }
5455 }
5456 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5457 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5458 }
5459 }
5460 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5464 || (head != op.head
5465 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5466 .await?
5467 .is_empty())
5468 || (!git::is_clean(&winner.worktree).await?
5469 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5470 .await
5471 .is_ok_and(|o| o.ok()))
5472 || !replay_proven
5473 {
5474 let why = format!(
5475 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5476 op.id
5477 );
5478 let sync = self
5479 .state
5480 .base_sync
5481 .as_mut()
5482 .context("reserved synchronization has no lifetime accounting")?;
5483 sync.conflict = Some(why.clone());
5484 self.state.event("land", why);
5485 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5486 .await?;
5487 return Ok(());
5488 }
5489 if head != op.head {
5490 git::sync_to_head(&winner.worktree).await?;
5491 }
5492 tip = op.tip.clone();
5493 }
5494 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5495 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5496 if pending.is_none()
5497 && !conflicted
5498 && fetched_base
5499 && self
5500 .state
5501 .base_sync_recovery
5502 .episodes
5503 .last()
5504 .is_some_and(|e| e.boundary.is_some())
5505 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5506 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5507 && owner.runs.last() == Some(&self.state.id)
5508 && owner.status != crate::queue::TaskStatus::Done
5509 && !owner.fresh_start
5510 && owner.review_branch.is_none()
5511 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5512 && let Some(approval) = owner.resumption.as_ref()
5513 && approval.instruction == owner.instruction
5514 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5515 && git::is_clean(&winner.worktree).await?
5516 && self.state.renew_base_episode(&task, approval, &head, &tip)
5517 {
5518 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5519 self.state.save()?;
5520 }
5521
5522 if (behind > 0 || conflicted || head == tip)
5531 && self
5532 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5533 .await?
5534 {
5535 return Ok(());
5536 }
5537 if conflicted && self.state.base_recovery_pending() {
5538 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5539 let owner =
5540 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5541 let owner_valid = owner.as_ref().is_ok_and(|t| {
5542 t.runs.last() == Some(&self.state.id)
5543 && t.status != crate::queue::TaskStatus::Done
5544 && !t.fresh_start
5545 && t.review_branch.is_none()
5546 });
5547 let invalidation = if !owner_valid {
5548 Some("latest task ownership changed")
5549 } else if !owner.as_ref().is_ok_and(|t| {
5550 authorization
5551 .generation
5552 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5553 }) {
5554 Some("owner approval generation changed")
5555 } else if let Some(scope) = &authorization.scope {
5556 if scope.tip != tip {
5557 Some("fetched base changed")
5558 } else if !self
5559 .state
5560 .base_sync_recovery
5561 .episodes
5562 .last()
5563 .is_some_and(|e| e.id == scope.episode)
5564 {
5565 Some("active synchronization interval changed")
5566 } else if !owner
5567 .as_ref()
5568 .is_ok_and(|t| t.instruction == scope.instruction)
5569 {
5570 Some("task instruction changed")
5571 } else {
5572 None
5573 }
5574 } else {
5575 None
5576 };
5577 if let Some(reason) = invalidation {
5578 let authorization = self
5579 .state
5580 .base_sync_recovery
5581 .authorizations
5582 .last_mut()
5583 .unwrap();
5584 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5585 self.state.save()?;
5586 return Ok(());
5587 }
5588 if !fetched_base
5589 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590 || !git::is_clean(&winner.worktree).await?
5591 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5592 {
5593 return Ok(());
5594 }
5595 }
5596 let recovery = conflicted && self.state.consume_base_recovery(&head);
5597 if conflicted {
5598 self.state.save()?;
5600 if !recovery {
5601 return Ok(());
5602 }
5603 }
5604
5605 if behind == 0 {
5606 if let Some(from) = self
5615 .state
5616 .rebase_fixes
5617 .iter()
5618 .rev()
5619 .find_map(|r| r.from.clone())
5620 && from != head
5621 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5622 .await
5623 .is_ok_and(|o| o.ok())
5624 {
5625 git::sync_to_head(&winner.worktree).await?;
5626 }
5627 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5633 if let Some(why) = &conflict {
5634 self.state.status = RunStatus::Blocked;
5635 self.state.event("land", why.clone());
5636 }
5637 self.state.base_sync = Some(BaseSync {
5638 tip,
5639 behind: 0,
5640 attempts,
5641 conflict,
5642 already_in: None,
5643 });
5644 if pending.is_some() {
5645 self.state.finish_base_operation(Some(&head));
5646 }
5647 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5648 .await?;
5649 self.state.save()?;
5650 return Ok(());
5651 }
5652
5653 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5654 let why = format!(
5655 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5656 synchronization(s); synchronizing again would only race it",
5657 winner.branch
5658 );
5659 self.state.status = RunStatus::Blocked;
5660 self.state.base_sync = Some(BaseSync {
5661 tip,
5662 behind,
5663 attempts,
5664 conflict: Some(why.clone()),
5665 already_in: None,
5666 });
5667 self.state.event("land", why);
5668 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5669 .await?;
5670 self.state.save()?;
5671 return Ok(());
5672 }
5673
5674 if self.state.foreign_checkout.is_some() {
5675 let wait = repository_wait(&self.state, None);
5678 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5679 if self.state.base_sync.is_none() {
5680 self.state.base_sync = Some(BaseSync {
5681 tip: tip.clone(),
5682 behind,
5683 attempts: 0,
5684 conflict: None,
5685 already_in: None,
5686 });
5687 }
5688 self.state.reserve_base_operation(&head, &tip);
5689 self.state.save()?;
5690 let merged = git::git_raw(
5691 &winner.worktree,
5692 &[
5693 "-c",
5694 "core.hooksPath=/dev/null",
5695 "merge",
5696 "--no-ff",
5697 "--no-autostash",
5698 "--no-edit",
5699 "-m",
5700 &format!("Merge {tracking} before verifying"),
5701 &tip,
5702 ],
5703 )
5704 .await?;
5705 let conflict = if merged.ok() {
5706 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5707 self.state.event(
5708 "land",
5709 format!("merged {tracking} into {} before verifying", winner.branch),
5710 );
5711 None
5712 } else {
5713 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5714 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5715 }
5716 let why = format!(
5717 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5718 winner.branch,
5719 merged.stderr.chars().take(600).collect::<String>()
5720 );
5721 self.state.status = RunStatus::Blocked;
5722 self.state.event("land", &why);
5723 Some(why)
5724 };
5725 let synced = conflict.is_none();
5726 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5727 self.state
5728 .finish_base_operation(synced.then_some(fresh.as_str()));
5729 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5730 self.state.base_sync = Some(BaseSync {
5731 tip,
5732 behind: if synced { 0 } else { behind },
5733 attempts,
5734 conflict,
5735 already_in: None,
5736 });
5737 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5738 .await?;
5739 self.state.save()?;
5740 return Ok(());
5741 }
5742
5743 self.state.event(
5744 "land",
5745 format!(
5746 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5747 winner.branch
5748 ),
5749 );
5750 self.state.save()?;
5751
5752 let branch_tracking = format!("{remote}/{}", winner.branch);
5758 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5759 let remote_tip = if self.state.task_pr_branch().is_none()
5760 && matches!(&fetched_branch, Ok(o) if o.ok())
5761 {
5762 git::rev_parse(&repo, &branch_tracking).await.ok()
5763 } else {
5764 None
5765 };
5766 if let Some(theirs) = &remote_tip
5770 && !git::is_ancestor(&repo, theirs, &head).await
5771 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5772 .await
5773 .is_ok_and(|missing| missing.is_empty())
5774 {
5775 let why = format!(
5776 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5777 them",
5778 short(theirs),
5779 winner.branch
5780 );
5781 self.state.status = RunStatus::Blocked;
5782 self.state.base_sync = Some(BaseSync {
5783 tip,
5784 behind,
5785 attempts,
5786 conflict: Some(why.clone()),
5787 already_in: None,
5788 });
5789 self.state.event("land", why);
5790 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5791 .await?;
5792 self.state.save()?;
5793 return Ok(());
5794 }
5795
5796 if self.state.base_sync.is_none() {
5797 self.state.base_sync = Some(BaseSync {
5798 tip: tip.clone(),
5799 behind,
5800 attempts: 0,
5801 conflict: None,
5802 already_in: None,
5803 });
5804 }
5805 self.state.reserve_base_operation(&head, &tip);
5806 self.state.save()?;
5807 let scratch = self.state.dir().join("base-sync");
5808 let rebased =
5809 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5810 .await
5811 {
5812 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5813 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5814 Err(e) => Err(e),
5815 };
5816 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5817 let stop_kind = match &rebased {
5818 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5819 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5820 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5821 };
5822 match rebased {
5823 Ok(None) => {
5824 git::sync_to_head(&winner.worktree).await?;
5828 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5829 self.state.finish_base_operation(Some(&fresh));
5830 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5831 let mut conflict = None;
5832 if let Some(pinned) = &remote_tip {
5833 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5834 match pushed {
5835 Ok(o) if o.ok() => self.state.event(
5836 "land",
5837 format!("pushed rebased {} to {remote}", winner.branch),
5838 ),
5839 Ok(o) => {
5840 conflict = Some(format!(
5841 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5842 winner.branch,
5843 short(pinned),
5844 o.stderr.chars().take(600).collect::<String>()
5845 ));
5846 }
5847 Err(e) => {
5848 conflict = Some(format!(
5849 "rebased {} locally but could not push it: {e:#}",
5850 winner.branch
5851 ));
5852 }
5853 }
5854 }
5855 if let Some(why) = &conflict {
5856 self.state.status = RunStatus::Blocked;
5857 self.state.event("land", why.clone());
5858 }
5859 self.state.base_sync = Some(BaseSync {
5860 tip: tip.clone(),
5861 behind: 0,
5862 attempts,
5863 conflict,
5864 already_in: None,
5865 });
5866 self.state
5867 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5868 }
5869 Ok(Some(conflict)) => {
5870 self.state.finish_base_operation(None);
5871 let why = format!(
5872 "{} conflicts with {tracking} and did not rebase: {}",
5873 winner.branch,
5874 conflict.chars().take(600).collect::<String>()
5875 );
5876 self.state.status = RunStatus::Blocked;
5877 self.state.base_sync = Some(BaseSync {
5878 tip,
5879 behind,
5880 attempts,
5881 conflict: Some(why.clone()),
5882 already_in: None,
5883 });
5884 self.state.event("land", why);
5885 }
5886 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5890 self.state.save()?;
5891 return Ok(());
5892 }
5893 Err(e) => {
5894 self.state.finish_base_operation(None);
5895 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5896 self.state.status = RunStatus::Blocked;
5897 self.state.base_sync = Some(BaseSync {
5898 tip,
5899 behind,
5900 attempts,
5901 conflict: Some(why.clone()),
5902 already_in: None,
5903 });
5904 self.state.event("land", why);
5905 }
5906 }
5907 if !self.state.base_sync_blocked() {
5908 self.state.base_sync_recovery.stop = None;
5909 }
5910 self.hold_base_sync_kind(stop_kind).await?;
5911 self.state.save()?;
5912 Ok(())
5913 }
5914
5915 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5921 if self.state.task_pr_branch().is_some() {
5924 return None;
5925 }
5926 let repo = self.state.repo.clone();
5927 let remote = self.state.config.merge.remote.clone();
5928 let fetched = git::fetch(&repo, &remote, branch).await;
5929 if !matches!(&fetched, Ok(o) if o.ok()) {
5930 return None;
5931 }
5932 let branch_tracking = format!("{remote}/{branch}");
5933 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5934 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5935 return None;
5936 }
5937 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5938 .await
5939 .is_ok_and(|missing| missing.is_empty())
5940 {
5941 return Some(format!(
5942 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5943 them",
5944 short(&theirs)
5945 ));
5946 }
5947 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5948 Ok(o) if o.ok() => {
5949 self.state
5950 .event("land", format!("pushed rebased {branch} to {remote}"));
5951 None
5952 }
5953 Ok(o) => Some(format!(
5954 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5955 someone may have pushed): {}",
5956 short(&theirs),
5957 o.stderr.chars().take(600).collect::<String>()
5958 )),
5959 Err(e) => Some(format!(
5960 "{branch} is rebased locally but could not be pushed: {e:#}"
5961 )),
5962 }
5963 }
5964
5965 async fn settle_already_in(
5982 &mut self,
5983 branch: &str,
5984 tip: &str,
5985 head: &str,
5986 attempts: usize,
5987 behind: usize,
5988 ) -> Result<bool> {
5989 let repo = self.state.repo.clone();
5990 let remote = self.state.config.merge.remote.clone();
5991 let start = self.state.base_commit.clone();
5992 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5993 Ok(Some(e)) => e,
5994 Ok(None) => return Ok(false),
5995 Err(e) => {
5996 tracing::warn!("already-in-base check for {branch}: {e:#}");
5997 return Ok(false);
5998 }
5999 };
6000 let mut verified = vec![head.to_owned()];
6001 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6002 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6003 if matches!(&fetched, Ok(o) if o.ok())
6004 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6005 && theirs != head
6006 {
6007 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6008 Ok(Some(_)) => verified.push(theirs),
6009 _ => return Ok(false),
6010 }
6011 }
6012 let base_branch = self.state.base_branch.clone();
6013 let message = format!(
6014 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6015 land",
6016 evidence.names(),
6017 evidence.proof.as_str()
6018 );
6019 let closed =
6020 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6021 .await;
6022 match closed {
6023 Ok(Ok(url)) => self
6024 .state
6025 .event("land", format!("closed {url}: superseded on {base_branch}")),
6026 Ok(Err(why)) => self
6027 .state
6028 .event("land", format!("did not close a pull request: {why}")),
6029 Err(e) => {
6030 let why = format!(
6031 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6032 not be closed ({e:#}); resume to retry"
6033 );
6034 self.state.status = RunStatus::Blocked;
6035 self.state.base_sync = Some(BaseSync {
6036 tip: tip.to_owned(),
6037 behind,
6038 attempts,
6039 conflict: Some(why.clone()),
6040 already_in: None,
6041 });
6042 self.state.event("land", why);
6043 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6044 .await?;
6045 self.state.save()?;
6046 return Ok(true);
6047 }
6048 }
6049 self.state.status = RunStatus::AlreadyInBase;
6050 self.state.base_sync_recovery.stop = None;
6051 self.state.base_sync = Some(BaseSync {
6052 tip: tip.to_owned(),
6053 behind,
6054 attempts,
6055 conflict: None,
6056 already_in: Some(evidence),
6057 });
6058 self.state.event("land", message);
6059 self.state.save()?;
6060 self.settle_questions();
6061 Ok(true)
6062 }
6063
6064 fn landing_base(&self) -> String {
6074 self.state
6075 .base_sync
6076 .as_ref()
6077 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6078 }
6079
6080 pub async fn fix_selected(
6113 &mut self,
6114 ids: &[String],
6115 reason: &str,
6116 allow_stale: bool,
6117 ) -> Result<()> {
6118 let reason = reason.trim();
6119 if reason.is_empty() {
6120 bail!("a fix request needs a reason — that is the operator's own record of why");
6121 }
6122 if ids.is_empty() {
6123 bail!("no finding id given");
6124 }
6125 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6126 bail!(
6127 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6128 has already concluded — can be given a targeted fix. A run still \
6129 in progress should simply be resumed; a `merged` run's branch has \
6130 already landed, so its answer is a fresh `magi review <branch>`, \
6131 not reopening this run's own record",
6132 self.state.id,
6133 self.state.status.as_str()
6134 );
6135 }
6136 let Some(winner) = self.state.winner().cloned() else {
6137 bail!("run {} has no winning candidate to fix", self.state.id);
6138 };
6139 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6140 bail!(
6141 "branch `{}` no longer exists; this run cannot be extended",
6142 winner.branch
6143 );
6144 }
6145 let home = crate::run::home();
6146 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6147 bail!(
6148 "run {} is currently being worked on by another magi process",
6149 self.state.id
6150 );
6151 }
6152 let _claim = FixClaim::acquire(&self.state.dir())?;
6158
6159 let mut seen = BTreeSet::new();
6163 let mut findings = Vec::new();
6164 let mut missing = Vec::new();
6165 for id in ids {
6166 if !seen.insert(id.clone()) {
6167 continue;
6168 }
6169 match self.state.finding(id) {
6170 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6171 id: f.id.clone(),
6172 severity: f.severity,
6173 reviewer_vote: rec.vote,
6174 round: round.round,
6175 round_head: round.head.clone(),
6176 reviewer: rec.reviewer,
6177 agent: rec.agent.clone(),
6178 file: f.file.clone(),
6179 line: f.line,
6180 title: f.title.clone(),
6181 detail: f.detail.clone(),
6182 outcome: OperatorFixOutcome::Pending,
6183 }),
6184 None => missing.push(id.clone()),
6185 }
6186 }
6187 if !missing.is_empty() {
6188 bail!(
6189 "unknown finding id(s): {}; nothing was changed",
6190 missing.join(", ")
6191 );
6192 }
6193
6194 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6195 let stale_details: Vec<(String, String)> = findings
6196 .iter()
6197 .filter(|f| f.round_head != head_at_request)
6198 .map(|f| (f.id.clone(), f.round_head.clone()))
6199 .collect();
6200 let stale = !stale_details.is_empty();
6201 if stale && !allow_stale {
6202 bail!(
6203 "the branch has moved since some finding(s) were raised — {} — now \
6204 at {}; pass --allow-stale to fix anyway, or re-run review first",
6205 stale_details
6206 .iter()
6207 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6208 .collect::<Vec<_>>()
6209 .join(", "),
6210 short(&head_at_request)
6211 );
6212 }
6213
6214 let request = OperatorFixRequest {
6215 requested_at: Timestamp::now(),
6216 reason: reason.to_owned(),
6217 findings,
6218 head_at_request: head_at_request.clone(),
6219 allow_stale,
6220 stale,
6221 fix: None,
6222 result_head: None,
6223 follow_up_review_run: None,
6224 };
6225 self.state.event(
6226 "fix",
6227 format!(
6228 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6229 request.findings.len(),
6230 request
6231 .findings
6232 .iter()
6233 .map(|f| f.id.as_str())
6234 .collect::<Vec<_>>()
6235 .join(", "),
6236 ),
6237 );
6238 self.state.operator_fixes.push(request);
6245 self.state.save()?;
6246 let request_index = self.state.operator_fixes.len() - 1;
6247
6248 let repo = self.state.repo.clone();
6257 let coordination = self.coordinate(&repo).await?;
6258 if winner.worktree.exists() {
6259 let dirty = git::git(
6262 &winner.worktree,
6263 &["status", "--porcelain", "--untracked-files=all"],
6264 )
6265 .await?;
6266 let only_withheld = dirty.lines().all(|l| {
6267 l.strip_prefix("?? ")
6268 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6269 });
6270 if !only_withheld {
6271 bail!(
6272 "`{}` has uncommitted changes; refusing to touch it — commit or \
6273 discard them first",
6274 winner.worktree.display()
6275 );
6276 }
6277 git::worktree_remove(&self.state.repo, &winner.worktree)
6278 .await
6279 .ok();
6280 }
6281 let fix_worktree = self.state.worktree_root().join("operator-fix");
6282 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6283 git::git(
6284 &self.state.repo,
6285 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6286 )
6287 .await
6288 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6289 if !git::is_clean(&fix_worktree).await? {
6290 git::worktree_remove(&self.state.repo, &fix_worktree)
6291 .await
6292 .ok();
6293 bail!(
6294 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6295 winner.branch
6296 );
6297 }
6298 let repo_for_setup = self.state.repo.clone();
6299 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6300 Ok(()) => {
6301 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6302 }
6303 Err(e) => Err(e),
6304 };
6305 if let Err(e) = setup {
6306 git::worktree_remove(&self.state.repo, &fix_worktree)
6307 .await
6308 .ok();
6309 return Err(e);
6310 }
6311
6312 self.state.resource_block = None;
6315
6316 let run_id = self.state.id.clone();
6317 let prompts = self.state.config.prompts.clone();
6318 let language = self.state.config.graph.language.clone();
6319 let sessions = self.state.config.graph.sessions;
6320 let artifacts = agent::artifacts_dir(&self.state.dir());
6321 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6322 .findings
6323 .iter()
6324 .map(|f| Finding {
6325 id: f.id.clone(),
6326 severity: f.severity,
6327 file: f.file.clone(),
6328 line: f.line,
6329 title: f.title.clone(),
6330 detail: f.detail.clone(),
6331 })
6332 .collect();
6333 let fix_prompt = prompt::operator_fix_in(
6334 &self.state.instruction,
6335 &finding_list,
6336 reason,
6337 &stale_details,
6338 &head_at_request,
6339 &language,
6340 &self.state.config.graph.github_language,
6341 );
6342 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6343 drop(coordination);
6344 let (job, seat, out) = self
6345 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6346 prompt: fix_prompt.clone(),
6347 spec,
6348 seat,
6349 cwd: fix_worktree.clone(),
6350 timeout,
6351 allow_write: true,
6352 sessions,
6353 artifacts: artifacts.clone(),
6354 stem: "operator-fix".to_owned(),
6355 handover: None,
6356 })
6357 .await;
6358 let agent_id = seat.agent.clone();
6359
6360 let mut fix = FixRecord {
6361 agent: agent_id,
6362 addressed: Vec::new(),
6363 rejected: Vec::new(),
6364 notes: String::new(),
6365 committed: false,
6366 failed: None,
6367 duration_ms: 0,
6368 continuation: None,
6369 };
6370 let mut final_seat = seat.clone();
6371 match out {
6372 AgentOutcome::Ok(o) => {
6373 fix.duration_ms = o.duration_ms;
6374 let parsed = verdict::extract_json::<FixReport>(&o.text);
6375 let incomplete_reason = match &parsed {
6376 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6377 "the reply parsed, but it reported a command whose own CLI \
6378 never confirmed an exit status"
6379 .to_owned(),
6380 ),
6381 Ok(_) => None,
6382 Err(e) => Some(e.to_string()),
6383 };
6384 match incomplete_reason {
6385 None => {
6386 let report = parsed.expect("checked Ok above");
6387 fix.addressed = report.addressed;
6388 fix.rejected = report.rejected;
6389 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6390 }
6391 Some(reason) => {
6392 let (resumed_seat, resolved, failure, cont) = self
6393 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6394 .await;
6395 fix.duration_ms += cont.cumulative_wait_ms;
6396 fix.continuation = Some(cont);
6397 final_seat = resumed_seat;
6398 match resolved {
6399 Some(report) => {
6400 fix.addressed = report.addressed;
6401 fix.rejected = report.rejected;
6402 fix.notes =
6403 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6404 }
6405 None => fix.failed = failure,
6406 }
6407 }
6408 }
6409 }
6410 AgentOutcome::Dropped(o) => {
6411 fix.duration_ms = o.duration_ms;
6412 let why = o
6413 .dropped
6414 .as_ref()
6415 .map(|d| d.why.as_str())
6416 .unwrap_or("the CLI ended the stream without delivering its answer");
6417 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6418 }
6419 AgentOutcome::Quota(o) => {
6420 self.state.quota.push(QuotaLoss::from_report(
6421 final_seat.key.clone(),
6422 "fix",
6423 o.quota.as_ref(),
6424 ));
6425 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6426 }
6427 AgentOutcome::Failed(e) => fix.failed = Some(e),
6428 AgentOutcome::ResourceBlocked(e) => {
6429 fix.failed = Some(format!("the fixer was not started: {e}"));
6430 }
6431 }
6432 if fix.continuation.is_none() {
6435 fix.continuation = Some(ContinuationRecord::not_needed());
6436 }
6437 self.state.seats.insert(final_seat.key.clone(), final_seat);
6438
6439 let rescue_message = format!(
6440 "magi: operator-selected fix ({}) (uncommitted work)",
6441 self.state.operator_fixes[request_index]
6442 .findings
6443 .iter()
6444 .map(|f| f.id.as_str())
6445 .collect::<Vec<_>>()
6446 .join(", ")
6447 );
6448 let repo = self.state.repo.clone();
6449 let coordination = self.coordinate(&repo).await?;
6450 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6451 self.state.note_withheld("fix", &r.withheld);
6452 }
6453 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6454 fix.committed = after != head_at_request;
6455 git::worktree_remove(&self.state.repo, &fix_worktree)
6456 .await
6457 .ok();
6458 drop(coordination);
6459
6460 self.state.event(
6461 "fix",
6462 match &fix.failed {
6463 Some(reason) => format!(
6464 "operator fix: adoption report was lost ({reason}); {}",
6465 if fix.committed {
6466 "committed"
6467 } else {
6468 "NO new commit"
6469 }
6470 ),
6471 None => format!(
6472 "operator fix: {} addressed, {} rejected, {}",
6473 fix.addressed.len(),
6474 fix.rejected.len(),
6475 if fix.committed {
6476 "committed"
6477 } else {
6478 "NO new commit"
6479 }
6480 ),
6481 },
6482 );
6483
6484 for f in &mut self.state.operator_fixes[request_index].findings {
6491 f.outcome = if fix.failed.is_some() {
6492 OperatorFixOutcome::Unreported
6493 } else if fix.addressed.contains(&f.id) {
6494 OperatorFixOutcome::Addressed
6495 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6496 OperatorFixOutcome::Rejected { why: r.why.clone() }
6497 } else {
6498 OperatorFixOutcome::Unreported
6499 };
6500 }
6501
6502 let committed = fix.committed;
6503 if committed {
6504 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6505 }
6506 self.state.operator_fixes[request_index].fix = Some(fix);
6507 self.state.save()?;
6510 if self.state.resource_block.is_some() {
6511 return self.stop_for_resource();
6512 }
6513
6514 if committed {
6515 self.state.event(
6516 "fix",
6517 format!(
6518 "operator fix committed {}; opening a follow-up review-only run",
6519 short(&after)
6520 ),
6521 );
6522 let origin =
6525 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6526 match Self::review(
6527 &self.state.repo,
6528 &winner.branch,
6529 self.state.config.clone(),
6530 origin,
6531 )
6532 .await
6533 {
6534 Ok(mut follow_up) => {
6535 follow_up.state.event(
6536 "start",
6537 format!(
6538 "requested by an operator fix on run {} for finding(s) {}",
6539 self.state.id,
6540 self.state.operator_fixes[request_index]
6541 .findings
6542 .iter()
6543 .map(|f| f.id.as_str())
6544 .collect::<Vec<_>>()
6545 .join(", "),
6546 ),
6547 );
6548 follow_up.state.save()?;
6549 let follow_up_id = follow_up.state.id.clone();
6550 let adopted = match crate::direct::adopt(
6553 &follow_up.state,
6554 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6555 ) {
6556 Ok(a) => a,
6557 Err(e) => {
6558 self.state.event(
6561 "fix",
6562 format!(
6563 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6564 ),
6565 );
6566 self.state.save()?;
6567 return Ok(());
6568 }
6569 };
6570 let executed = follow_up.execute().await;
6571 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6572 if let Some(a) = adopted {
6573 a.finish(&follow_up.state, executed);
6574 }
6575 if let Some(e) = failure {
6576 self.state.event(
6577 "fix",
6578 format!(
6579 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6580 ),
6581 );
6582 }
6583 self.state.operator_fixes[request_index].follow_up_review_run =
6584 Some(follow_up_id);
6585 }
6586 Err(e) => {
6587 self.state.event(
6588 "fix",
6589 format!("committed the fix but could not open a follow-up review: {e:#}"),
6590 );
6591 }
6592 }
6593 self.state.save()?;
6594 }
6595
6596 Ok(())
6597 }
6598
6599 async fn ask_fixer(
6619 &mut self,
6620 winner: &Candidate,
6621 node: &'static str,
6622 round: Option<usize>,
6623 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6624 ) -> (SeatJob, SeatState, AgentOutcome) {
6625 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6626 self.ask_fixer_among(attempts, node, round, None, build)
6627 .await
6628 }
6629
6630 async fn ask_fixer_among(
6635 &mut self,
6636 attempts: Vec<(AgentSpec, String)>,
6637 node: &'static str,
6638 round: Option<usize>,
6639 restore: Option<(&Path, &str)>,
6640 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6641 ) -> (SeatJob, SeatState, AgentOutcome) {
6642 let run_id = self.state.id.clone();
6643 let prompts = self.state.config.prompts.clone();
6644 let cache = self.state.config.cache_dir();
6645 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6646 let mut last = None;
6647 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6648 let seat = self.seat(&key, &spec.id);
6649 let mut job = build(spec.clone(), seat);
6650 if i > 0 {
6651 job.stem = format!("{}-{}", job.stem, spec.id);
6652 }
6653 let ctx = WaveCtx {
6654 carry_seats: false,
6655 pauses: Some((&self.pause, &self.interrupt)),
6656 run: &run_id,
6657 node,
6658 prompts: &prompts,
6659 cache: cache.as_deref(),
6660 round,
6661 };
6662 let (seat, out) =
6663 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6664 let advances = match &out {
6667 AgentOutcome::Ok(o) => agent::output_advances(o),
6668 AgentOutcome::ResourceBlocked(_) => false,
6669 _ => true,
6670 };
6671 if let Some(next) = ids.get(i + 1)
6672 && advances
6673 {
6674 self.state.seats.insert(seat.key.clone(), seat.clone());
6675 let class =
6676 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6677 let reason = match &out {
6678 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6679 other => fail_reason(other),
6680 };
6681 if let Some((wt, rev)) = restore
6685 && let Err(e) = self.graft_restore(wt, rev).await
6686 {
6687 let why =
6688 format!("could not restore the worktree before the next fixer: {e:#}");
6689 tracing::warn!("{why}");
6690 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6691 AgentOutcome::ResourceBlocked(why)
6692 } else {
6693 AgentOutcome::Failed(why)
6694 };
6695 last = Some((job, seat, out));
6696 break;
6697 }
6698 record_handover(
6699 &mut self.state,
6700 node,
6701 &seat.key,
6702 &spec.id,
6703 next,
6704 &class,
6705 &reason,
6706 );
6707 continue;
6708 }
6709 last = Some((job, seat, out));
6710 break;
6711 }
6712 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6713 if let AgentOutcome::Quota(output) = &out
6714 && node != "graft"
6715 {
6716 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6721 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6722 self.state.quota.push(loss);
6723 self.state.seats.insert(seat.key.clone(), seat.clone());
6724 self.state.status = RunStatus::Blocked;
6725 return (
6726 job,
6727 seat,
6728 AgentOutcome::ResourceBlocked(
6729 "fixer unavailable; restore agent availability and resume".into(),
6730 ),
6731 );
6732 }
6733 (job, seat, out)
6734 }
6735
6736 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6747 use crate::recovery::{Mode, Repair};
6748 if !self.state.reviews.is_empty() {
6749 return Ok(());
6750 }
6751 let Some(mut rec) = self.state.recovery.clone() else {
6752 return Ok(());
6753 };
6754 if rec.mode != Mode::VerifyRepair {
6755 return Ok(());
6756 }
6757 let wt = winner.worktree.clone();
6758 let mut head = git::rev_parse(&wt, "HEAD").await?;
6759 match rec.repair.clone() {
6760 Repair::Done { .. } => return Ok(()),
6761 Repair::WaitingForRestore { before } => {
6762 self.graft_restore(&wt, &before).await?;
6763 head = before;
6764 rec.repair = Repair::NotStarted;
6765 self.state.recovery = Some(rec.clone());
6766 self.state.save()?;
6767 }
6768 Repair::Started { before } => {
6769 if let Err(e) = self.graft_restore(&wt, &before).await {
6773 if e.is::<crate::repo_lock::ResourceBlocked>() {
6774 return Err(e);
6775 }
6776 return self
6777 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6778 .await;
6779 }
6780 rec.repair = Repair::Done {
6781 before: before.clone(),
6782 after: before,
6783 committed: false,
6784 failed: Some("interrupted; rolled back".to_owned()),
6785 };
6786 self.state
6787 .event("recovery", "the repair was interrupted and rolled back");
6788 self.state.recovery = Some(rec);
6789 self.state.save()?;
6790 return Ok(());
6791 }
6792 Repair::NotStarted => {}
6793 }
6794 if crate::recovery::recheck(&mut rec, &head) {
6795 self.state.event("recovery", rec.summary());
6796 self.state.recovery = Some(rec);
6797 self.state.save()?;
6798 return Ok(());
6799 }
6800 rec.repair = Repair::Started {
6801 before: head.clone(),
6802 };
6803 self.state.recovery = Some(rec.clone());
6804 self.state.save()?;
6805
6806 let language = self.state.config.graph.language.clone();
6807 let sessions = self.state.config.graph.sessions;
6808 let artifacts = agent::artifacts_dir(&self.state.dir());
6809 let fix_prompt = prompt::recovery_fix(
6810 &self.state.instruction,
6811 &rec,
6812 self.state.config.graph.review_rounds,
6813 &language,
6814 );
6815 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6816 self.state.event(
6817 "recovery",
6818 format!(
6819 "repair: the fixer starts from the failed verification on {}",
6820 short(&head)
6821 ),
6822 );
6823 let (_job, seat, out) = self
6824 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6825 prompt: fix_prompt.clone(),
6826 spec,
6827 seat,
6828 cwd: wt.clone(),
6829 timeout,
6830 allow_write: true,
6831 sessions,
6832 artifacts: artifacts.clone(),
6833 stem: "fix-recovery".to_owned(),
6834 handover: None,
6835 })
6836 .await;
6837 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6838 rec.repair = Repair::WaitingForRestore { before: head };
6839 self.state.recovery = Some(rec);
6840 return self.stop_for_resource();
6841 }
6842 let failed = match &out {
6843 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6844 AgentOutcome::Quota(o) => {
6845 self.state.quota.push(QuotaLoss::from_report(
6846 seat.key.clone(),
6847 "fix",
6848 o.quota.as_ref(),
6849 ));
6850 Some("rate limited (quota); the fixer could not run".to_owned())
6851 }
6852 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6853 AgentOutcome::Failed(e) => Some(e.clone()),
6854 };
6855 self.state.seats.insert(seat.key.clone(), seat);
6856 let coordination = self.coordinate(&wt).await?;
6857 let failed = match failed {
6858 Some(why) => {
6860 if let Err(e) = git::restore_to(&wt, &head).await {
6861 return self
6862 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6863 .await;
6864 }
6865 Some(why)
6866 }
6867 None => {
6868 if let Ok(r) = git::rescue_commit(
6869 &wt,
6870 "magi: recovery repair of the failed verification (uncommitted work)",
6871 )
6872 .await
6873 {
6874 self.state.note_withheld("fix", &r.withheld);
6875 }
6876 None
6877 }
6878 };
6879 let after = git::rev_parse(&wt, "HEAD").await?;
6880 drop(coordination);
6881 self.state.event(
6882 "recovery",
6883 match &failed {
6884 Some(why) => format!("repair: nothing kept ({why})"),
6885 None if after != head => {
6886 format!("repair: committed {}", short(&after))
6887 }
6888 None => "repair: the fixer made no commit".to_owned(),
6889 },
6890 );
6891 rec.repair = Repair::Done {
6892 before: head.clone(),
6893 committed: after != head,
6894 after,
6895 failed,
6896 };
6897 self.state.recovery = Some(rec);
6898 self.state.save()?;
6899 Ok(())
6900 }
6901
6902 async fn stop_recovery(
6905 &mut self,
6906 mut rec: crate::recovery::RecoveryRecord,
6907 head: &str,
6908 why: String,
6909 ) -> Result<()> {
6910 rec.repair = crate::recovery::Repair::Started {
6912 before: head.to_owned(),
6913 };
6914 self.state.recovery = Some(rec);
6915 self.state
6916 .event("recovery", format!("repair could not be undone: {why}"));
6917 self.state.status = RunStatus::Blocked;
6918 self.state.save()?;
6919 bail!("recovery repair could not be rolled back: {why}")
6920 }
6921
6922 async fn review_loop(&mut self) -> Result<()> {
6923 if self.hold_base_sync().await? {
6924 return Ok(());
6925 }
6926 loop {
6927 if self
6928 .state
6929 .base_sync
6930 .as_ref()
6931 .is_some_and(|s| s.conflict.is_some())
6932 {
6933 return Ok(());
6934 }
6935 if self.catchup_repairable() {
6936 let record = self.state.reviews.last().expect("repairable round").clone();
6937 let winner = self
6938 .state
6939 .winner()
6940 .cloned()
6941 .context("catch-up repair has no winner")?;
6942 if !self.fix_review_round(&winner, record).await? {
6943 return Ok(());
6944 }
6945 }
6946 self.review_loop_pass().await?;
6947 if !self.catchup_repairable() {
6948 return Ok(());
6949 }
6950 }
6951 }
6952
6953 fn catchup_repairable(&self) -> bool {
6956 self.state.reviews.len() < self.state.config.graph.review_rounds
6957 && self.state.reviews.last().is_some_and(|r| {
6958 !r.clean
6959 && r.blocking == 0
6960 && r.fix.is_none()
6961 && r.e2e_status() == E2eStatus::Failed
6962 })
6963 }
6964
6965 async fn review_loop_pass(&mut self) -> Result<()> {
6966 if self
6971 .state
6972 .base_sync
6973 .as_ref()
6974 .is_some_and(|s| s.conflict.is_some())
6975 {
6976 return Ok(());
6977 }
6978 let run_id = self.state.id.clone();
6983 let prompts = self.state.config.prompts.clone();
6984 let Some(winner) = self.state.winner().cloned() else {
6985 return Ok(());
6986 };
6987 let granted_review = self
6988 .state
6989 .repair_grants
6990 .last()
6991 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6992 .cloned();
6993 let fresh_review = granted_review
6994 .as_ref()
6995 .is_some_and(|g| self.state.reviews.len() == g.review_start);
6996 let max_rounds = granted_review
6997 .as_ref()
6998 .map_or(self.state.config.graph.review_rounds, |g| {
6999 g.review_start.saturating_add(1)
7000 });
7001 if let Some(status) =
7011 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7012 {
7013 if status == RunStatus::Gating {
7016 self.record_contested_handoff();
7017 }
7018 self.state.status = status;
7019 self.state.save()?;
7020 return Ok(());
7021 }
7022 self.state.status = RunStatus::Reviewing;
7023 if !fresh_review
7033 && self.state.reviews.last().is_some_and(|r| {
7034 r.e2e_status() == E2eStatus::ResourceBlocked
7035 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7036 })
7037 {
7038 let shell = self.state.config.shell();
7039 return self
7040 .stop_unresolved_review(
7041 "the last round's own verification never resolved",
7042 &shell,
7043 &winner.worktree,
7044 )
7045 .await;
7046 }
7047
7048 if granted_review.is_none() {
7049 self.recovery_repair(&winner).await?;
7050 }
7051 if self.resource_halt()? {
7052 return Ok(());
7053 }
7054
7055 let repo = self.state.repo.clone();
7056 let root = self.state.worktree_root();
7057 let language = self.state.config.graph.language.clone();
7058 let sessions = self.state.config.graph.sessions;
7059 let artifacts = self.merge_repair_artifacts();
7060 let base = self.landing_base();
7061 let base_short = short(&base);
7062 let reviewers = self.roles.reviewers.clone();
7063 let shell = self.state.config.shell();
7064
7065 for round in (self.state.reviews.len() + 1)..=max_rounds {
7066 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7067 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7068 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7069 let prev_verification = self
7078 .state
7079 .reviews
7080 .last()
7081 .and_then(|r| r.verification_summary(&head));
7082 let prior_section = match (&self.state.recovery, round) {
7085 (Some(rec), 1) if self.state.reviews.is_empty() => {
7086 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7087 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7088 Some(crate::recovery::head_of(
7089 &git::diff(&winner.worktree, reviewed, "HEAD")
7090 .await
7091 .unwrap_or_default(),
7092 crate::recovery::SINCE_DIFF_MAX,
7093 ))
7094 }
7095 _ => None,
7096 };
7097 Some(prompt::recovery_review(rec, since.as_deref()))
7098 }
7099 _ => None,
7100 };
7101
7102 let mut jobs = Vec::new();
7106 let coordination = self.coordinate(&repo).await?;
7107 for (r, spec) in reviewers.iter().cloned().enumerate() {
7108 let wt = root.join(format!("review-{}", r + 1));
7109 if wt.exists() {
7112 git::reset_detached(&wt, &head).await?;
7113 } else {
7114 git::worktree_add_detached(&repo, &wt, &head).await?;
7115 }
7116 self.setup_seat_worktree(&repo, &wt, false).await?;
7117 let seat_key = format!("review-{}", r + 1);
7118 let spec = pick_start_spec(
7121 &self.roles.reviewer_roster,
7122 spec,
7123 self.state.seat_history.get(&seat_key),
7124 );
7125 let seat = self.seat(&seat_key, &spec.id);
7126 jobs.push(SeatJob {
7127 prompt: prompt::review(&prompt::ReviewCtx {
7128 instruction: &self.state.instruction,
7129 branch: &winner.branch,
7130 base_short: &base_short,
7131 stat: &stat,
7132 patch: &patch,
7133 verification: prev_verification.as_ref(),
7134 prior: prior_section.as_deref(),
7135 reviewers: reviewers.len(),
7136 round,
7137 rounds: max_rounds,
7138 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7141 lens: Lens::for_seat(r),
7142 language: &language,
7143 github_language: &self.state.config.graph.github_language,
7144 }),
7145 spec,
7146 seat,
7147 cwd: wt,
7148 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7149 allow_write: false,
7150 sessions,
7151 artifacts: artifacts.clone(),
7152 stem: format!("review-{round}-{}", r + 1),
7153 handover: None,
7154 });
7155 }
7156
7157 drop(coordination);
7158 self.state.event(
7159 "review",
7160 format!(
7161 "round {round}: {} reviewers on {}",
7162 jobs.len(),
7163 short(&head)
7164 ),
7165 );
7166 let mut quota_losses = Vec::new();
7167 let review_retries = self.state.config.graph.retries;
7168 let review_cache = self.state.config.cache_dir();
7169 let ctx = WaveCtx {
7170 carry_seats: true,
7171 pauses: Some((&self.pause, &self.interrupt)),
7172 run: &run_id,
7173 node: "review",
7174 prompts: &prompts,
7175 cache: review_cache.as_deref(),
7176 round: Some(round),
7177 };
7178 let results = ask_json_wave::<Review>(
7179 jobs,
7180 Arc::clone(&self.sem),
7181 review_retries,
7182 &self.roles.reviewer_roster,
7183 &ctx,
7184 &mut quota_losses,
7185 &mut self.state,
7186 &|_: &Review| Ok(()),
7187 )
7188 .await;
7189 if self.state.resource_block.is_some() {
7190 return self.stop_for_resource();
7191 }
7192 let round_quota_missing = quota_losses.len();
7196 self.state.quota.extend(quota_losses);
7197
7198 let mut records = Vec::new();
7199 let mut all_findings = Vec::new();
7200 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7201 let agent_id = seat.agent.clone();
7202 self.state.seats.insert(seat.key.clone(), seat);
7203 let mut record = ReviewRecord {
7204 reviewer: r + 1,
7205 agent: agent_id,
7206 summary: String::new(),
7207 findings: Vec::new(),
7208 vote: None,
7209 failed: None,
7210 duration_ms: 0,
7211 attempts,
7217 };
7218 match res {
7219 Ok((review, out)) => {
7220 record.summary =
7228 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7229 record.vote = Some(review.vote);
7230 record.duration_ms = out.duration_ms;
7231 for (n, mut f) in review.findings.into_iter().enumerate() {
7232 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7235 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7236 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7237 f.file = f
7243 .file
7244 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7245 all_findings.push(f.clone());
7246 record.findings.push(f);
7247 }
7248 self.state.event(
7249 "review",
7250 format!(
7251 "round {round}: reviewer {} voted {} with {} finding(s)",
7252 r + 1,
7253 review.vote.label(),
7254 record.findings.len()
7255 ),
7256 );
7257 }
7258 Err(e) => {
7259 record.failed = Some(e.to_string());
7260 self.state.event(
7261 "review",
7262 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7263 );
7264 }
7265 }
7266 records.push(record);
7267 }
7268
7269 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7276 let vote_split =
7277 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7278 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7279 if vote_split {
7280 self.state.event(
7281 "review",
7282 format!(
7283 "round {round}: votes split ({}) — one round of reconsideration",
7284 initial_votes
7285 .iter()
7286 .map(|v| v.label())
7287 .collect::<Vec<_>>()
7288 .join(", ")
7289 ),
7290 );
7291 let panel: Vec<ReviewSeatReport<'_>> = records
7294 .iter()
7295 .filter_map(|r| {
7296 r.vote.map(|vote| ReviewSeatReport {
7297 reviewer: r.reviewer,
7298 vote,
7299 summary: &r.summary,
7300 findings: &r.findings,
7301 })
7302 })
7303 .collect();
7304
7305 let mut jobs = Vec::new();
7306 let mut seats_at = Vec::new();
7307 for (r, spec) in reviewers.iter().cloned().enumerate() {
7308 if records[r].vote.is_none() {
7312 continue;
7313 }
7314 let wt = root.join(format!("review-{}", r + 1));
7315 let seat_key = format!("review-{}", r + 1);
7316 let spec = self.occupant(&seat_key, spec);
7317 let seat = self.seat(&seat_key, &spec.id);
7318 let build = |with_patch: bool| {
7326 prompt::review_reconsider(&ReviewReconsiderCtx {
7327 instruction: &self.state.instruction,
7328 reviewer: r + 1,
7329 lens: Lens::for_seat(r),
7330 panel: &panel,
7331 patch: with_patch.then_some(ReviewPatch {
7332 branch: &winner.branch,
7333 base_short: &base_short,
7334 stat: &stat,
7335 patch: &patch,
7336 }),
7337 round,
7338 rounds: max_rounds,
7339 language: &language,
7340 })
7341 };
7342 let full = build(true);
7343 let prompt = if has_context(&spec, &seat, sessions) {
7344 build(false)
7345 } else {
7346 full.clone()
7347 };
7348 jobs.push(SeatJob {
7349 prompt,
7350 spec,
7351 seat,
7352 cwd: wt,
7353 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7354 allow_write: false,
7355 sessions,
7356 artifacts: artifacts.clone(),
7357 stem: format!("review-{round}-reconsider-{}", r + 1),
7358 handover: Some(full),
7359 });
7360 seats_at.push(r);
7361 }
7362
7363 let mut recon_quota_losses = Vec::new();
7364 let recon_cache = self.state.config.cache_dir();
7365 let recon_ctx = WaveCtx {
7366 carry_seats: true,
7367 pauses: Some((&self.pause, &self.interrupt)),
7368 run: &run_id,
7369 node: "review",
7370 prompts: &prompts,
7371 cache: recon_cache.as_deref(),
7372 round: Some(round),
7373 };
7374 let recon_results = ask_json_wave::<ReviewRevote>(
7375 jobs,
7376 Arc::clone(&self.sem),
7377 review_retries,
7378 &self.roles.reviewer_roster,
7379 &recon_ctx,
7380 &mut recon_quota_losses,
7381 &mut self.state,
7382 &|_: &ReviewRevote| Ok(()),
7383 )
7384 .await;
7385 if self.state.resource_block.is_some() {
7386 return self.stop_for_resource();
7387 }
7388 self.state.quota.extend(recon_quota_losses);
7389
7390 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7391 let agent_id = seat.agent.clone();
7392 self.state.seats.insert(seat.key.clone(), seat);
7393 let mut rec = ReviewRevoteRecord {
7394 reviewer: r + 1,
7395 agent: agent_id,
7396 vote: None,
7397 reason: String::new(),
7398 failed: None,
7399 };
7400 match res {
7401 Ok((rv, _)) => {
7402 rec.vote = Some(rv.vote);
7403 rec.reason =
7404 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7405 self.state.event(
7406 "review",
7407 format!(
7408 "round {round}: reviewer {} revoted {}",
7409 r + 1,
7410 rv.vote.label()
7411 ),
7412 );
7413 }
7414 Err(e) => {
7415 rec.failed = Some(e.to_string());
7416 self.state.event(
7417 "review",
7418 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7419 );
7420 }
7421 }
7422 reconsideration.push(rec);
7423 }
7424 } else if initial_votes.len() > 1 {
7425 self.state.event(
7426 "review",
7427 format!(
7428 "round {round}: votes agreed ({}) — no reconsideration",
7429 initial_votes[0].label()
7430 ),
7431 );
7432 }
7433
7434 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7435 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7436 let defer_e2e =
7447 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7448 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7449 let reason =
7450 format!("{blocking} blocking finding(s) already required a fix this round");
7451 self.state.event(
7452 "verify",
7453 format!(
7454 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7455 {}); it will run once a round has none left",
7456 short(&head)
7457 ),
7458 );
7459 (Vec::new(), false, true, Some(reason))
7460 } else {
7461 let e2e_commands = self.state.config.verify.e2e.clone();
7462 let cache_dir = self.state.config.cache_dir();
7463 let context = format!("round {round}");
7464 let wait = self.repository_wait();
7465 let (e2e, verify_retried) = with_cache_lease(
7466 &mut self.state,
7467 cache_dir.as_deref(),
7468 "e2e",
7469 "e2e",
7470 &winner.worktree,
7471 &head,
7472 verify_timeout,
7473 &context,
7474 &wait,
7475 |state, budget| {
7476 let shell = shell.clone();
7477 let e2e_commands = e2e_commands.clone();
7478 let worktree = winner.worktree.clone();
7479 let context = context.clone();
7480 async move {
7481 run_e2e_with_retry(
7482 state,
7483 &shell,
7484 &e2e_commands,
7485 &worktree,
7486 budget,
7487 &context,
7488 )
7489 .await
7490 }
7491 },
7492 )
7493 .await?;
7494 (e2e, verify_retried, false, None)
7495 };
7496
7497 let expected = records.len();
7498 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7499 let incomplete = answered < expected;
7500 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7501 let policy = self.state.config.graph.incomplete_review;
7502 let clean = round_is_clean(
7503 blocking,
7504 e2e_ok,
7505 answered,
7506 expected,
7507 round_quota_missing,
7508 policy,
7509 );
7510
7511 let mut round_record = ReviewRound {
7512 round,
7513 head: head.clone(),
7514 verified_head: None,
7515 verified_at: None,
7516 reviews: records,
7517 e2e,
7518 verify_retried,
7519 e2e_deferred,
7520 e2e_defer_reason,
7521 fix: None,
7522 blocking,
7523 answered,
7524 expected,
7525 clean,
7526 progressed: false,
7527 vote_split,
7528 reconsideration,
7529 verdict: None,
7530 };
7531 round_record.verdict = ReviewVote::worst(
7535 round_record
7536 .final_votes()
7537 .into_iter()
7538 .map(|(_, _, vote)| vote),
7539 );
7540 if !matches!(
7551 round_record.e2e_status(),
7552 E2eStatus::Deferred | E2eStatus::NotConfigured
7553 ) {
7554 round_record.verified_head = Some(head.clone());
7555 round_record.verified_at = Some(Timestamp::now());
7556 }
7557
7558 if incomplete {
7559 let missing: Vec<String> = round_record
7560 .reviews
7561 .iter()
7562 .filter(|r| r.failed.is_some())
7563 .map(|r| format!("review-{}", r.reviewer))
7564 .collect();
7565 self.state.event(
7566 "review",
7567 format!(
7568 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7569 missing.join(", ")
7570 ),
7571 );
7572 }
7573
7574 if clean {
7575 self.state.event(
7576 "review",
7577 if incomplete && policy == IncompleteReviewPolicy::Warn {
7578 format!(
7579 "round {round}: clean (warn policy, incomplete panel) — no \
7580 blocking findings from the seats that answered, verification green"
7581 )
7582 } else if incomplete {
7583 format!(
7584 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7585 quorum) — no blocking findings from the seats that answered, \
7586 verification green",
7587 expected - answered
7588 )
7589 } else {
7590 format!("round {round}: clean — no blocking findings, verification green")
7591 },
7592 );
7593 self.state.reviews.push(round_record);
7594 self.state.status = RunStatus::Gating;
7595 self.state.save()?;
7596 return Ok(());
7597 }
7598
7599 if incomplete && blocking == 0 && e2e_ok {
7607 self.state.reviews.push(round_record);
7608 self.state.save()?;
7609 if round == max_rounds {
7610 self.state.status = RunStatus::Blocked;
7611 self.state.event(
7612 "review",
7613 format!(
7614 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7615 refusing to call it clean",
7616 expected - answered
7617 ),
7618 );
7619 return Ok(());
7620 }
7621 continue;
7622 }
7623
7624 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7636 self.state.reviews.push(round_record);
7637 return self
7638 .stop_unresolved_review(
7639 "the round's own verification could not run",
7640 &shell,
7641 &winner.worktree,
7642 )
7643 .await;
7644 }
7645
7646 if round == max_rounds {
7647 self.state.reviews.push(round_record);
7648 return self
7649 .stop_reviewing(
7650 &format!(
7651 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7652 ),
7653 &shell,
7654 &winner.worktree,
7655 )
7656 .await;
7657 }
7658
7659 if !self.fix_review_round(&winner, round_record).await? {
7660 return Ok(());
7661 }
7662 }
7663 Ok(())
7664 }
7665
7666 async fn fix_review_round(
7669 &mut self,
7670 winner: &Candidate,
7671 mut round_record: ReviewRound,
7672 ) -> Result<bool> {
7673 let round = round_record.round;
7674 let shell = self.state.config.shell();
7675 let step = self.run_fix_round(winner, &mut round_record).await?;
7676 if matches!(step, FixStep::Resource) {
7677 self.stop_for_resource()?;
7678 return Ok(false);
7679 }
7680 if self.state.reviews.len() < round {
7681 self.state.reviews.push(round_record);
7682 } else {
7683 self.state.reviews[round - 1] = round_record;
7684 }
7685 self.state.save()?;
7686
7687 if matches!(step, FixStep::Recorded { report_lost: true }) {
7700 self.stop_reviewing(
7701 "the fixer's adoption report never came back, even after resuming its \
7702 own seat; refusing to start another round against the same worktree \
7703 while that is unresolved",
7704 &shell,
7705 &winner.worktree,
7706 )
7707 .await?;
7708 return Ok(false);
7709 }
7710
7711 let streak = self
7712 .state
7713 .reviews
7714 .iter()
7715 .rev()
7716 .take_while(|r| !r.progressed)
7717 .count();
7718 if streak >= STAGNANT_LIMIT {
7719 self.stop_reviewing(
7720 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7721 &shell,
7722 &winner.worktree,
7723 )
7724 .await?;
7725 return Ok(false);
7726 }
7727 Ok(true)
7728 }
7729
7730 async fn run_fix_round(
7732 &mut self,
7733 winner: &Candidate,
7734 round_record: &mut ReviewRound,
7735 ) -> Result<FixStep> {
7736 let round = round_record.round;
7737 let max_rounds = self.state.config.graph.review_rounds;
7738 let run_id = self.state.id.clone();
7739 let prompts = self.state.config.prompts.clone();
7740 let language = self.state.config.graph.language.clone();
7741 let sessions = self.state.config.graph.sessions;
7742 let artifacts = self.merge_repair_artifacts();
7743 let base = self.landing_base();
7744 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7745 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7746 let this_round_verification = round_record.verification_summary(&head);
7747 let all_findings: Vec<_> = round_record
7748 .reviews
7749 .iter()
7750 .flat_map(|r| r.findings.iter().cloned())
7751 .collect();
7752 let blocking_findings: Vec<_> = all_findings
7755 .iter()
7756 .filter(|f| f.severity.blocks())
7757 .cloned()
7758 .collect();
7759 let fix_prompt = prompt::fix_in(
7760 &self.state.instruction,
7761 &blocking_findings,
7762 this_round_verification.as_ref(),
7763 round,
7764 max_rounds,
7765 &language,
7766 &self.state.config.graph.github_language,
7767 );
7768 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7769 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7770 let (job, seat, out) = self
7771 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7772 prompt: fix_prompt.clone(),
7773 spec,
7774 seat,
7775 cwd: winner.worktree.clone(),
7776 timeout,
7777 allow_write: true,
7778 sessions,
7779 artifacts: artifacts.clone(),
7780 stem: format!("fix-{round}"),
7781 handover: None,
7782 })
7783 .await;
7784 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7787 return Ok(FixStep::Resource);
7788 }
7789 let agent_id = seat.agent.clone();
7790
7791 let mut fix = FixRecord {
7792 agent: agent_id,
7793 addressed: Vec::new(),
7794 rejected: Vec::new(),
7795 notes: String::new(),
7796 committed: false,
7797 failed: None,
7798 duration_ms: 0,
7799 continuation: None,
7800 };
7801 let mut continuation = ContinuationRecord::not_needed();
7802 let mut final_seat = seat.clone();
7803 match out {
7804 AgentOutcome::Ok(o) => {
7805 fix.duration_ms = o.duration_ms;
7806 let parsed = verdict::extract_json::<FixReport>(&o.text);
7807 let incomplete_reason = match &parsed {
7814 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7815 "the reply parsed, but it reported a command whose own CLI never \
7816 confirmed an exit status"
7817 .to_owned(),
7818 ),
7819 Ok(_) => None,
7820 Err(e) => Some(e.to_string()),
7821 };
7822 match incomplete_reason {
7823 None => {
7824 let report = parsed.expect("checked Ok above");
7825 fix.addressed = report.addressed;
7826 fix.rejected = report.rejected;
7827 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7828 }
7829 Some(reason) => {
7830 let (resumed_seat, resolved, failure, cont) = self
7831 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7832 .await;
7833 fix.duration_ms += cont.cumulative_wait_ms;
7834 continuation = cont;
7835 final_seat = resumed_seat;
7836 match resolved {
7837 Some(report) => {
7838 fix.addressed = report.addressed;
7839 fix.rejected = report.rejected;
7840 fix.notes =
7841 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7842 }
7843 None => fix.failed = failure,
7844 }
7845 }
7846 }
7847 }
7848 AgentOutcome::Dropped(o) => {
7850 fix.duration_ms = o.duration_ms;
7851 let why = o
7852 .dropped
7853 .as_ref()
7854 .map(|d| d.why.as_str())
7855 .unwrap_or("the CLI ended the stream without delivering its answer");
7856 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7857 }
7858 AgentOutcome::Quota(o) => {
7859 self.state.quota.push(QuotaLoss::from_report(
7860 final_seat.key.clone(),
7861 "fix",
7862 o.quota.as_ref(),
7863 ));
7864 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7865 }
7866 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7867 }
7868 if self.state.resource_block.is_some() {
7869 self.state.seats.insert(final_seat.key.clone(), final_seat);
7870 return Ok(FixStep::Resource);
7871 }
7872 fix.continuation = Some(continuation);
7873 self.state.seats.insert(final_seat.key.clone(), final_seat);
7874 let coordination = self.coordinate(&winner.worktree).await?;
7875 if let Ok(r) = git::rescue_commit(
7876 &winner.worktree,
7877 &format!("magi: review round {round} fixes (uncommitted work)"),
7878 )
7879 .await
7880 {
7881 self.state.note_withheld("fix", &r.withheld);
7882 }
7883 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7884 fix.committed = after != before;
7885 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7893 drop(coordination);
7894 let progressed = diff_after != patch;
7895 let commit_note = if fix.committed {
7896 "committed"
7897 } else {
7898 "NO new commit"
7899 };
7900 let tree_note = if progressed {
7901 "changed vs base"
7902 } else {
7903 "unchanged vs base"
7904 };
7905 self.state.event(
7906 "fix",
7907 match &fix.failed {
7908 Some(reason) => {
7914 format!(
7915 "round {round}: fixer's adoption report was lost ({reason}); \
7916 {commit_note}, tree {tree_note}"
7917 )
7918 }
7919 None => format!(
7920 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7921 {tree_note}{}",
7922 fix.addressed.len(),
7923 fix.rejected.len(),
7924 if continuation.outcome == ContinuationOutcome::Resumed {
7925 format!(
7926 " (adoption report recovered after {} continuation(s))",
7927 continuation.attempts
7928 )
7929 } else {
7930 String::new()
7931 },
7932 ),
7933 },
7934 );
7935 round_record.fix = Some(fix);
7936 round_record.progressed = progressed;
7937 Ok(FixStep::Recorded {
7938 report_lost: matches!(
7939 continuation.outcome,
7940 ContinuationOutcome::Exhausted
7941 | ContinuationOutcome::QuotaLost
7942 | ContinuationOutcome::NoSession
7943 ),
7944 })
7945 }
7946
7947 fn record_contested_handoff(&mut self) {
7981 if self.state.contested_handoff.is_some() {
7982 return;
7983 }
7984 let Some(contested) = self
7985 .state
7986 .reviews
7987 .last()
7988 .and_then(ReviewRound::contested_handoff)
7989 else {
7990 return;
7991 };
7992 self.state.event(
7993 "review",
7994 format!(
7995 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7996 wait for the owner's approval",
7997 contested.findings.len(),
7998 contested.rejecters.len()
7999 ),
8000 );
8001 self.state.contested_handoff = Some(contested);
8002 }
8003
8004 async fn stop_unresolved_review(
8005 &mut self,
8006 why: &str,
8007 shell: &[String],
8008 worktree: &Path,
8009 ) -> Result<()> {
8010 self.stop_reviewing_inner(why, shell, worktree, true).await
8011 }
8012
8013 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8014 self.stop_reviewing_inner(why, shell, worktree, false).await
8015 }
8016
8017 async fn stop_reviewing_inner(
8018 &mut self,
8019 why: &str,
8020 shell: &[String],
8021 worktree: &Path,
8022 repairable: bool,
8023 ) -> Result<()> {
8024 let round_idx = self.state.reviews.len() - 1;
8025 let needs_catchup_run = matches!(
8033 self.state.reviews[round_idx].e2e_status(),
8034 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8035 );
8036 if needs_catchup_run {
8037 let round = self.state.reviews[round_idx].round;
8038 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8039 let commands = self.state.config.verify.e2e.clone();
8040 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8041 let cache_dir = self.state.config.cache_dir();
8042 let context = format!(
8043 "round {round}: verification unresolved, catching up before the final decision"
8044 );
8045 let wait = self.repository_wait();
8046 let (outcomes, verify_retried) = with_cache_lease(
8047 &mut self.state,
8048 cache_dir.as_deref(),
8049 "e2e",
8050 "e2e",
8051 worktree,
8052 &attempted_head,
8053 timeout,
8054 &context,
8055 &wait,
8056 |state, budget| {
8057 let shell = shell.to_vec();
8058 let commands = commands.clone();
8059 let context = context.clone();
8060 async move {
8061 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8062 .await
8063 }
8064 },
8065 )
8066 .await?;
8067 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8068 self.state.record_base_progress(&attempted_head);
8069 }
8070 let last = &mut self.state.reviews[round_idx];
8071 last.e2e = outcomes;
8072 last.verify_retried = verify_retried;
8073 last.verified_head = Some(attempted_head);
8080 last.verified_at = Some(Timestamp::now());
8081 if verify_inconclusive(&last.e2e) {
8082 self.state.save()?;
8089 return Ok(());
8090 }
8091 last.e2e_deferred = false;
8092 if last.answered == last.expected {
8097 last.clean = round_is_clean(
8098 last.blocking,
8099 last.e2e.iter().all(CommandOutcome::ok),
8100 last.answered,
8101 last.expected,
8102 0,
8103 self.state.config.graph.incomplete_review,
8104 );
8105 }
8106 }
8107 let last = &self.state.reviews[round_idx];
8108 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8109
8110 match last.e2e_status() {
8111 E2eStatus::Failed => {
8112 let red: Vec<String> = last
8113 .e2e
8114 .iter()
8115 .filter(|o| !o.ok())
8116 .map(|o| {
8117 format!(
8118 "`{}` -> {:?}\n{}",
8119 o.command,
8120 o.code,
8121 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8122 )
8123 })
8124 .collect();
8125 let can_repair = last.blocking == 0 && last.fix.is_none();
8126 self.state.reviews[round_idx].clean = false;
8127 self.state
8128 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8129 self.state.status = RunStatus::Blocked;
8130 if repairable
8131 && can_repair
8132 && self.state.reviews.len() < self.state.config.graph.review_rounds
8133 {
8134 self.state.status = RunStatus::Reviewing;
8135 }
8136 }
8137 E2eStatus::ResourceBlocked => {
8142 self.state.event(
8143 "review",
8144 format!(
8145 "{why}; e2e could not run (resource unavailable); not \
8146 deciding yet"
8147 ),
8148 );
8149 }
8150 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8151 self.state.event(
8152 "review",
8153 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8154 );
8155 self.record_contested_handoff();
8156 self.state.status = RunStatus::Gating;
8157 }
8158 }
8159 self.state.save()?;
8160 Ok(())
8161 }
8162
8163 async fn gate(&mut self) -> Result<()> {
8166 if self.hold_base_sync().await? {
8167 return Ok(());
8168 }
8169 if self.state.status == RunStatus::Failed
8181 || self
8182 .state
8183 .base_sync
8184 .as_ref()
8185 .is_some_and(|s| s.conflict.is_some())
8186 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8187 != Some(RunStatus::Gating)
8188 {
8189 return Ok(());
8190 }
8191 let Some(winner) = self.state.winner().cloned() else {
8192 return Ok(());
8193 };
8194 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8195 self.state.reconcile_verification_head(&head);
8196 self.state.save()?;
8197 let wait = self.repository_wait();
8198 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8199 return Ok(());
8200 }
8201 if self.state.gate_ran {
8202 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8213 self.state.status = RunStatus::Blocked;
8214 self.state.save()?;
8215 }
8216 return Ok(());
8217 }
8218 self.state.status = RunStatus::Gating;
8219 let mut outcomes = self.run_gate(&winner).await?;
8220 let mut attempted_head = self.state.gate_head.clone();
8221 loop {
8222 if verify_inconclusive(&outcomes) {
8233 self.state.save()?;
8234 return Ok(());
8235 }
8236 if outcomes.iter().all(CommandOutcome::ok) {
8237 break;
8238 }
8239 let fixes_before = self.state.gate_fixes.len();
8240 match self.gate_fix_round(&winner, &outcomes).await? {
8241 GateFix::Retry => {
8242 outcomes = self.run_gate(&winner).await?;
8243 attempted_head = self.state.gate_head.clone();
8244 }
8245 GateFix::Stop => {
8246 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8247 if self.state.gate_fixes.len() > fixes_before
8248 && attempted_head.as_deref() != Some(current.as_str())
8249 && verify_current_head(&mut self.state, &winner, &wait).await?
8250 {
8251 outcomes = self.run_gate(&winner).await?;
8254 attempted_head = self.state.gate_head.clone();
8255 } else {
8256 break;
8257 }
8258 }
8259 GateFix::Defer => {
8260 self.state.save()?;
8261 return Ok(());
8262 }
8263 }
8264 }
8265 let passed = outcomes.iter().all(CommandOutcome::ok);
8266 self.state.gate = outcomes;
8267 self.state.gate_ran = true;
8268 self.state.gate_head = attempted_head;
8269 if !self.state.gate.is_empty()
8270 && !verify_inconclusive(&self.state.gate)
8271 && let Some(head) = self.state.gate_head.clone()
8272 {
8273 self.state.record_base_progress(&head);
8274 }
8275 self.state.save()?;
8276 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8277 return Ok(());
8278 }
8279 if !passed {
8280 self.state.status = RunStatus::Blocked;
8281 let spent = self.state.gate_fixes.len();
8282 self.state.event(
8283 "gate",
8284 if spent == 0 {
8285 "gate failed; not merging".to_owned()
8286 } else {
8287 format!("gate failed after {spent} gate-fix round(s); not merging")
8288 },
8289 );
8290 }
8291 self.state.save()?;
8292 Ok(())
8293 }
8294
8295 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8305 let commands = state.config.verify.pre_gate.clone();
8306 if commands.is_empty() {
8307 return;
8308 }
8309 let shell = state.config.shell();
8310 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8311 let (outcomes, _) = run_commands(
8312 state,
8313 "pre_gate",
8314 "pre_gate",
8315 0,
8316 &shell,
8317 &commands,
8318 &winner.worktree,
8319 timeout,
8320 )
8321 .await;
8322 for o in &outcomes {
8323 if !o.ok() {
8324 tracing::warn!(
8325 "pre_gate `{}` failed ({:?}); the gate decides",
8326 o.command,
8327 o.code
8328 );
8329 }
8330 state.event(
8331 "pre_gate",
8332 format!(
8333 "`{}` -> {}",
8334 o.command,
8335 if o.ok() {
8336 "pass".to_owned()
8337 } else {
8338 format!(
8339 "FAIL ({:?})\n{}",
8340 o.code,
8341 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8342 )
8343 }
8344 ),
8345 );
8346 }
8347 state.pre_gate = outcomes;
8348 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8349 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8350 Ok(head) => {
8351 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8352 state.pre_gate_commit = Some(head);
8353 }
8354 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8355 },
8356 Ok(false) => {}
8357 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8358 }
8359 if let Err(e) = state.save() {
8360 tracing::warn!("could not persist the pre_gate record: {e:#}");
8361 }
8362 }
8363
8364 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8367 let wait = self.repository_wait();
8368 run_landing_gate(&mut self.state, winner, &wait).await
8369 }
8370
8371 async fn gate_fix_round(
8384 &mut self,
8385 winner: &Candidate,
8386 outcomes: &[CommandOutcome],
8387 ) -> Result<GateFix> {
8388 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8389 g.phase == crate::repair_budget::Phase::Fixing
8390 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8391 }) {
8392 self.state.gate_fixes.len().saturating_add(1)
8393 } else {
8394 self.state.config.graph.gate_fix_rounds
8395 };
8396 let spent = self.state.gate_fixes.len();
8397 if spent >= cap {
8398 if cap > 0 {
8399 self.state.event(
8400 "gate",
8401 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8402 );
8403 }
8404 return Ok(GateFix::Stop);
8405 }
8406 if !gate_fixable(outcomes) {
8407 self.state.event(
8408 "gate",
8409 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8410 command or similar); not spending a fix round on it",
8411 );
8412 return Ok(GateFix::Stop);
8413 }
8414 let min_free = self.state.config.disk.min_free_bytes;
8415 if min_free > 0 {
8416 match crate::disk::free_bytes(&winner.worktree) {
8417 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8418 Ok(free) => {
8419 self.state.event(
8420 "gate",
8421 format!(
8422 "only {free} bytes free ({min_free} required by `[disk] \
8423 min_free_bytes`); not spending a fix round on a failure the disk \
8424 may explain"
8425 ),
8426 );
8427 return Ok(GateFix::Stop);
8428 }
8429 Err(e) => {
8430 self.state.event(
8431 "gate",
8432 format!("free disk space could not be measured ({e:#}); no fix round"),
8433 );
8434 return Ok(GateFix::Stop);
8435 }
8436 }
8437 }
8438
8439 let attempt = spent + 1;
8440 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8441 let base = self.landing_base();
8442 let fix_prompt = prompt::gate_fix_in(
8443 &self.state.instruction,
8444 &failed,
8445 attempt,
8446 cap,
8447 &self.state.config.graph.language,
8448 &self.state.config.graph.github_language,
8449 );
8450 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8451 let sessions = self.state.config.graph.sessions;
8452 let artifacts = self.merge_repair_artifacts();
8453 self.state.event(
8454 "gate",
8455 format!("gate failed; gate-fix round {attempt} of {cap}"),
8456 );
8457 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8458 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8459 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8462 head: before.clone(),
8463 passed: false,
8464 outcomes: Vec::new(),
8465 });
8466 self.state.save()?;
8467 let (_, seat, out) = self
8468 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8469 prompt: fix_prompt.clone(),
8470 spec,
8471 seat,
8472 cwd: winner.worktree.clone(),
8473 timeout,
8474 allow_write: true,
8475 sessions,
8476 artifacts: artifacts.clone(),
8477 stem: format!("gate-fix-{attempt}"),
8478 handover: None,
8479 })
8480 .await;
8481 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8484 return Ok(GateFix::Defer);
8485 }
8486 let mut record = GateFixRecord {
8487 agent: seat.agent.clone(),
8488 failed,
8489 notes: String::new(),
8490 committed: false,
8491 error: None,
8492 };
8493 match out {
8494 AgentOutcome::Ok(o) => {
8495 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8498 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8499 }
8500 }
8501 AgentOutcome::Dropped(_) => {
8502 record.error = Some("the CLI dropped the stream".to_owned());
8503 }
8504 AgentOutcome::Quota(o) => {
8505 self.state.quota.push(QuotaLoss::from_report(
8506 seat.key.clone(),
8507 "gate-fix",
8508 o.quota.as_ref(),
8509 ));
8510 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8511 }
8512 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8513 }
8514 self.state.seats.insert(seat.key.clone(), seat);
8515 let coordination = self.coordinate(&winner.worktree).await?;
8516 if let Ok(r) = git::rescue_commit(
8517 &winner.worktree,
8518 &format!("magi: gate fix {attempt} (uncommitted work)"),
8519 )
8520 .await
8521 {
8522 self.state.note_withheld("gate-fix", &r.withheld);
8523 }
8524 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8525 record.committed = after != before;
8526 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8527 let note = record.error.clone();
8528 self.state.gate_fixes.push(record);
8529 self.state.save()?;
8530 if !changed {
8531 self.state.event(
8532 "gate",
8533 match note {
8534 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8535 None => format!("gate-fix round {attempt}: the tree did not change"),
8536 },
8537 );
8538 return Ok(GateFix::Stop);
8539 }
8540 self.state.event(
8541 "gate",
8542 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8543 );
8544
8545 drop(coordination);
8546 let wait = self.repository_wait();
8547 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8548 }
8549
8550 async fn guarded_pr_message(
8557 &mut self,
8558 winner: &Candidate,
8559 facts: Option<&BranchFacts>,
8560 posting: bool,
8561 ) -> Result<Option<PrMessage>> {
8562 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8563 let head = match final_head.as_deref() {
8564 Some(h) => crate::pr_outcome::FinalHead::At(h),
8565 None => crate::pr_outcome::FinalHead::Unknown,
8566 };
8567 let summarized = if posting {
8569 Some(self.summarize_change(winner, final_head.as_deref()).await)
8570 } else {
8571 None
8572 };
8573 if self.state.resource_block.is_some() {
8578 self.state.event(
8579 "pr-summary",
8580 "a shared resource was busy; nothing was published, resume to summarize again",
8581 );
8582 return Ok(None);
8583 }
8584 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8585 anyhow::bail!(
8586 "the final commit moved during public description generation; retry before publishing"
8587 );
8588 }
8589 let note = match (&summarized, &self.state.change_summary) {
8590 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8591 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8592 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8593 (None, _) => ChangeNote::NotAsked,
8594 };
8595 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8596 if !posting {
8597 let identity = crate::scrub::Identity::current();
8598 return Ok(Some(PrMessage {
8599 title: crate::scrub::scrub(&pr.title, &identity),
8600 body: crate::scrub::scrub(&pr.body, &identity),
8601 }));
8602 }
8603 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8606 if let Some(known) = self
8607 .state
8608 .github_text
8609 .clone()
8610 .filter(|g| g.fingerprint == raw_fp)
8611 {
8612 return self.settle_withheld(winner, &pr, known).await;
8613 }
8614 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8615 let mut violations =
8616 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8617 let identity = crate::scrub::Identity::current();
8618 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8619 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8620 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8621 {
8622 violations.push(crate::github_text::Violation::SensitiveData);
8623 }
8624 let has_language = violations
8628 .iter()
8629 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8630 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8631 self.state.event(
8632 "github-text",
8633 if has_language {
8634 format!("description rejected: {violations:?}; requesting one rewrite")
8635 } else {
8636 format!("description rejected: {violations:?}; asking the owner")
8637 },
8638 );
8639 let mut rewritten = false;
8640 let mut rejected_rewrite = None;
8641 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8642 {
8643 let output = text;
8644 if let Some(title) = summary_title(&output) {
8645 let raw_body = summary_without_title(&output);
8648 let clean_title = crate::scrub::scrub(&title, &identity);
8649 let mut body = crate::scrub::scrub(&raw_body, &identity);
8650 let redacted = clean_title != title || body != raw_body;
8651 let title = clean_title;
8652 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8653 if !complete {
8654 self.state.event(
8655 "github-text",
8656 "description rewrite omitted required findings",
8657 );
8658 }
8659 if !body.trim().is_empty() {
8660 body.push_str("\n\n");
8661 body.push_str(&public_pr_evidence(&self.state, head));
8662 }
8663 let marker = format!("magi:run/{}", self.state.id);
8664 if !body.trim().is_empty() && !body.contains(&marker) {
8665 body.push_str(&format!("\n\n{marker}"));
8666 }
8667 let rewrite = PrMessage {
8668 title: title.clone(),
8669 body: body.clone(),
8670 };
8671 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8672 let mut failed = crate::github_text::check_for(
8673 &self.state.config.graph,
8674 &title,
8675 &body,
8676 redecision,
8677 );
8678 if (!crate::github_text::shareable(&title)
8679 || !crate::github_text::shareable(&body))
8680 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8681 {
8682 failed.push(crate::github_text::Violation::SensitiveData);
8683 }
8684 if complete
8685 && !title.trim().is_empty()
8686 && !body.trim().is_empty()
8687 && !failed.is_empty()
8688 {
8689 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8690 title: title.clone(),
8691 body: body.clone(),
8692 failed: failed
8693 .iter()
8694 .map(|v| crate::github_text::category(*v).to_owned())
8695 .collect(),
8696 });
8697 }
8698 if complete
8699 && !title.trim().is_empty()
8700 && !raw_body.trim().is_empty()
8701 && failed.is_empty()
8702 {
8703 decision = redecision;
8704 pr = PrMessage { title, body };
8705 rewritten = true;
8706 self.state
8707 .event("github-text", "description rewrite passed");
8708 if redacted {
8709 self.state.event(
8710 "github-text",
8711 "sensitive data removed from description rewrite",
8712 );
8713 }
8714 }
8715 }
8716 }
8717 if !rewritten {
8718 self.state.event(
8719 "github-text",
8720 "description rewrite unavailable or rejected; asking the owner",
8721 );
8722 if let Some(w) =
8723 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8724 {
8725 self.retire_github_text_question("the withheld text changed");
8728 let record = crate::run::GithubTextAsk {
8729 rewrite: rejected_rewrite,
8730 fingerprint: raw_fp,
8731 title: w.title,
8732 body: w.body,
8733 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8734 question: None,
8735 asks: 0,
8736 resolved: false,
8737 chosen_title: None,
8738 choice: None,
8739 };
8740 return self.settle_withheld(winner, &pr, record).await;
8741 }
8742 }
8743 }
8744 let (title, body) =
8745 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8746 Ok(Some(PrMessage { title, body }))
8747 }
8748
8749 async fn summarize_change(
8760 &mut self,
8761 winner: &Candidate,
8762 head: Option<&str>,
8763 ) -> std::result::Result<(), String> {
8764 let Some(head) = head else {
8765 return Err("the final commit could not be read".to_owned());
8766 };
8767 let remote = self.state.config.merge.remote.clone();
8768 let tracking = format!("{remote}/{}", self.state.base_branch);
8769 if !git::rev_exists(&winner.worktree, &tracking).await {
8770 return Err(format!("the remote base `{tracking}` could not be read"));
8771 }
8772 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8773 Ok(b) if !b.is_empty() => b,
8774 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8775 };
8776 let evidence = crate::pr_outcome::summary_evidence(
8777 &self.state,
8778 crate::pr_outcome::FinalHead::At(head),
8779 );
8780 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8781 if self
8782 .state
8783 .change_summary
8784 .as_ref()
8785 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8786 {
8787 return Ok(());
8788 }
8789 if self.state.change_summary.take().is_some() {
8790 self.state.event(
8791 "pr-summary",
8792 "the final diff or evidence changed; the earlier change summary was discarded",
8793 );
8794 }
8795 if let Err(e) = write_artifact(
8797 &self.state,
8798 &format!("public-description-{fingerprint}.md"),
8799 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8800 ) {
8801 tracing::warn!("could not save public description evidence: {e:#}");
8802 }
8803 let chain = match agent::pick_chain(
8804 &self.state.config.agents,
8805 self.state.config.roles.synthesizer.as_ref(),
8806 &agent::installed,
8807 "synthesizer",
8808 ) {
8809 Ok(chain) => chain,
8810 Err(e) => {
8811 tracing::warn!("no agent for the change summary: {e:#}");
8812 self.state.event("pr-summary", "no summary agent available");
8813 return Err("no agent was available to summarize it".to_owned());
8814 }
8815 };
8816 let subjects = git::subjects(&winner.worktree, &base, head)
8817 .await
8818 .unwrap_or_default();
8819 let stat = git::diff_stat(&winner.worktree, &base, head)
8820 .await
8821 .unwrap_or_default();
8822 let stat: String = stat.chars().take(6_000).collect();
8823 let mut prompt = crate::prompt::change_summary_in(
8824 &base,
8825 head,
8826 &subjects,
8827 stat.trim(),
8828 &self.state.config.graph.github_language,
8829 );
8830 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8831 let sessions = self.state.config.graph.sessions;
8832 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8833 let prompts = self.state.config.prompts.clone();
8834 let cache = self.state.config.cache_dir();
8835 let run = self.state.id.clone();
8836 for (i, spec) in chain.into_iter().enumerate() {
8837 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8838 let github_spec = match agent::for_github_text(
8839 &spec,
8840 &self.state.config.graph.github_language,
8841 &winner.worktree,
8842 ) {
8843 Ok(spec) => spec,
8844 Err(e) => {
8845 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8846 continue;
8847 }
8848 };
8849 let job = SeatJob {
8850 spec: github_spec,
8851 seat,
8852 cwd: winner.worktree.clone(),
8853 prompt: prompt.clone(),
8854 timeout,
8855 allow_write: false,
8856 sessions,
8857 artifacts: agent::artifacts_dir(&self.state.dir()),
8858 stem: if i == 0 {
8859 "change-summary".to_owned()
8860 } else {
8861 format!("change-summary-{}", spec.id)
8862 },
8863 handover: None,
8864 };
8865 let ctx = WaveCtx {
8866 run: &run,
8867 node: "pr-summary",
8868 prompts: &prompts,
8869 cache: cache.as_deref(),
8870 round: None,
8871 carry_seats: false,
8872 pauses: Some((&self.pause, &self.interrupt)),
8873 };
8874 let (seat, outcome) =
8875 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8876 self.state.seats.insert(seat.key.clone(), seat);
8877 let text = match outcome {
8878 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8879 AgentOutcome::ResourceBlocked(why) => {
8880 if self.state.resource_block.is_none() {
8883 self.state.resource_block = Some(crate::run::ResourceBlock {
8884 cause: crate::run::ResourceBlockCause::Other,
8885 node: "pr-summary".to_owned(),
8886 kind: "shared resource".to_owned(),
8887 detail: why,
8888 at: Timestamp::now(),
8889 });
8890 }
8891 return Err("a shared resource was busy".to_owned());
8892 }
8893 _ => continue,
8894 };
8895 let blind = &self.state.config.blind;
8896 let Some(title) = summary_title(&text) else {
8897 continue;
8898 };
8899 let body: String = summary_without_title(&text)
8900 .lines()
8901 .filter(|l| !l.contains("magi:run/"))
8902 .collect::<Vec<_>>()
8903 .join("\n");
8904 let body = blind::sanitize_prose(body.trim(), blind);
8905 let title = blind::sanitize_prose(&title, blind);
8906 if body.trim().is_empty() || title.trim().is_empty() {
8907 continue;
8908 }
8909 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8910 continue;
8911 }
8912 if git::rev_parse(&winner.worktree, "HEAD")
8914 .await
8915 .ok()
8916 .as_deref()
8917 != Some(head)
8918 {
8919 return Err("the branch moved while the summary was written".to_owned());
8920 }
8921 if git::merge_base(&winner.worktree, &tracking, head)
8922 .await
8923 .ok()
8924 .as_deref()
8925 != Some(base.as_str())
8926 || crate::pr_outcome::summary_evidence(
8927 &self.state,
8928 crate::pr_outcome::FinalHead::At(head),
8929 ) != evidence
8930 {
8931 return Err("the summary evidence changed while it was written".to_owned());
8932 }
8933 self.state.change_summary = Some(crate::run::ChangeSummary {
8934 head: head.to_owned(),
8935 base,
8936 evidence: fingerprint,
8937 title,
8938 body,
8939 });
8940 self.state.event(
8941 "pr-summary",
8942 format!("change summary written by {} for the final diff", spec.id),
8943 );
8944 if let Err(e) = self.state.save() {
8945 tracing::warn!("could not save the change summary: {e:#}");
8946 }
8947 return Ok(());
8948 }
8949 self.state
8950 .event("pr-summary", "no agent produced a change summary");
8951 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8952 }
8953
8954 async fn rewrite_pr_text(
8964 &mut self,
8965 winner: &Candidate,
8966 pr: &PrMessage,
8967 violations: &[crate::github_text::Violation],
8968 ) -> Option<String> {
8969 let prompt = format!(
8970 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8971 pr.title, pr.body
8972 );
8973 let sessions = self.state.config.graph.sessions;
8974 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8975 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8978 let key = format!("impl-{}", winner.label);
8979 if let Some(spec) = self
8980 .state
8981 .config
8982 .agents
8983 .iter()
8984 .find(|a| a.id == winner.agent)
8985 .cloned()
8986 && let Some(existing) = self.state.seats.get(&key)
8987 && existing.agent == spec.id
8988 && agent::has_session(spec.kind, existing, sessions)
8989 {
8990 attempts.push((spec, Some(existing.clone())));
8991 }
8992 match agent::pick_chain(
8993 &self.state.config.agents,
8994 self.state.config.roles.language_judge.as_ref(),
8995 &agent::installed,
8996 "language_judge",
8997 ) {
8998 Ok(chain) => {
8999 for spec in chain {
9000 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9001 continue;
9002 }
9003 attempts.push((spec, None));
9004 }
9005 }
9006 Err(e) if attempts.is_empty() => {
9007 tracing::warn!("no agent for the description rewrite: {e:#}");
9008 self.state
9009 .event("github-text", "no rewrite agent available");
9010 return None;
9011 }
9012 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9013 }
9014 let prompts = self.state.config.prompts.clone();
9015 let cache = self.state.config.cache_dir();
9016 let run = self.state.id.clone();
9017 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9018 let resumed = resumed_seat.is_some();
9019 let seat = resumed_seat.unwrap_or_else(|| {
9020 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9021 });
9022 let github_spec = match agent::for_github_text(
9023 &spec,
9024 &self.state.config.graph.github_language,
9025 &winner.worktree,
9026 ) {
9027 Ok(spec) => spec,
9028 Err(e) => {
9029 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9030 continue;
9031 }
9032 };
9033 let job = SeatJob {
9034 spec: github_spec,
9035 seat,
9036 cwd: winner.worktree.clone(),
9037 prompt: prompt.clone(),
9038 timeout: retry_budget(timeout_full, resumed),
9039 allow_write: false,
9040 sessions,
9041 artifacts: agent::artifacts_dir(&self.state.dir()),
9042 stem: if i == 0 {
9043 "github-text-rewrite".to_owned()
9044 } else {
9045 format!("github-text-rewrite-{}", spec.id)
9046 },
9047 handover: None,
9048 };
9049 let ctx = WaveCtx {
9050 run: &run,
9051 node: "github-text",
9052 prompts: &prompts,
9053 cache: cache.as_deref(),
9054 round: None,
9055 carry_seats: false,
9056 pauses: Some((&self.pause, &self.interrupt)),
9057 };
9058 let (seat, outcome) =
9059 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9060 self.state.seats.insert(seat.key.clone(), seat);
9061 match outcome {
9062 AgentOutcome::Ok(o)
9063 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9064 {
9065 return Some(o.text);
9066 }
9067 AgentOutcome::ResourceBlocked(_) => return None,
9068 _ => {}
9069 }
9070 }
9071 None
9072 }
9073
9074 fn retire_github_text_question(&mut self, why: &str) {
9076 let Some(id) = self
9077 .state
9078 .github_text
9079 .as_ref()
9080 .and_then(|g| g.question.clone())
9081 else {
9082 return;
9083 };
9084 let _ = ask::Questions::open().update(&id, |q| {
9085 q.abandon(why);
9086 Ok(())
9087 });
9088 }
9089
9090 async fn settle_withheld(
9094 &mut self,
9095 winner: &Candidate,
9096 pr: &PrMessage,
9097 mut g: crate::run::GithubTextAsk,
9098 ) -> Result<Option<PrMessage>> {
9099 use crate::github_text as gt;
9100 let store = ask::Questions::open();
9101 let timeout = self.state.config.graph.answer_timeout;
9102 if !g.resolved {
9103 let mut standing = g
9104 .question
9105 .as_ref()
9106 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9107 if let Some(q) = standing.as_ref()
9108 && q.status.open()
9109 && gt::expired(q, timeout)
9110 {
9111 let why = format!("no answer within {}s of asking", timeout.max(1));
9112 standing = store
9113 .update(&q.id, |q| {
9114 q.abandon(&why);
9115 Ok(())
9116 })
9117 .ok()
9118 .map(|(q, ())| q);
9119 }
9120 let mut ask_again: Option<Option<Vec<String>>> = None;
9121 match standing {
9122 None if g.asks <= 1 => {
9125 g.asks = 0;
9126 ask_again = Some(None);
9127 }
9128 None => {
9129 g.resolved = true;
9130 g.chosen_title = None;
9131 g.choice = Some(GithubTextChoice::Fallback);
9132 }
9133 Some(q) if q.status.open() => {
9134 self.park_on_github_text(&g)?;
9135 return Ok(None);
9136 }
9137 Some(q) => match gt::read_reply(&q) {
9138 gt::Reply::Fallback => {
9139 g.resolved = true;
9140 g.chosen_title = None;
9141 g.choice = Some(GithubTextChoice::Fallback);
9142 }
9143 gt::Reply::Original => {
9144 g.resolved = true;
9145 g.chosen_title = None;
9146 g.choice = Some(GithubTextChoice::Original);
9147 }
9148 gt::Reply::Rewrite => {
9149 g.resolved = true;
9150 g.chosen_title = None;
9151 g.choice = Some(if g.rewrite.is_some() {
9152 GithubTextChoice::Rewrite
9153 } else {
9154 GithubTextChoice::Fallback
9155 });
9156 }
9157 gt::Reply::Title(text) => {
9158 let first = text
9159 .lines()
9160 .map(str::trim)
9161 .find(|l| !l.is_empty())
9162 .unwrap_or("");
9163 let decision = if first.is_empty() {
9164 None
9165 } else {
9166 let candidate = PrMessage {
9167 title: first.to_owned(),
9168 body: String::new(),
9169 };
9170 self.judge_pr_language(&winner.worktree, &candidate).await
9171 };
9172 match gt::vet_title(
9173 &text,
9174 decision,
9175 self.state.config.graph.github_english(),
9176 ) {
9177 Ok(title) => {
9178 g.resolved = true;
9179 g.chosen_title = Some(title);
9180 g.choice = Some(GithubTextChoice::MyTitle);
9181 }
9182 Err(cats) if g.asks < 2 => {
9183 ask_again = Some(Some(
9184 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9185 ))
9186 }
9187 Err(_) => {
9188 g.resolved = true;
9189 g.chosen_title = None;
9190 g.choice = Some(GithubTextChoice::Fallback);
9191 }
9192 }
9193 }
9194 },
9195 }
9196 if let Some(rejected) = ask_again {
9197 let w = gt::Withheld {
9200 title: g.title,
9201 body: g.body,
9202 categories: g.categories.clone(),
9203 };
9204 let artifact = match crate::run::write_artifact(
9207 &self.state,
9208 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9209 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9210 ) {
9211 Ok(p) => Some(p),
9212 Err(e) => {
9213 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9214 None
9215 }
9216 };
9217 let (detail, comparison) = gt::question_content(
9218 &w,
9219 &pr.title,
9220 &pr.body,
9221 &self.state.id,
9222 rejected.as_deref(),
9223 artifact.as_deref(),
9224 g.rewrite.as_ref(),
9225 );
9226 let mut q = ask::Question::new(
9227 self.state.id.clone(),
9228 gt::ASK_NODE.to_owned(),
9229 gt::ASK_SEAT.to_owned(),
9230 gt::question_summary(&self.state.config.graph.language, &w),
9231 detail,
9232 gt::question_choices(&w, g.rewrite.as_ref()),
9233 );
9234 q.posting_comparison = Some(comparison);
9235 q.answer_timeout = timeout;
9236 g.question = Some(q.id.clone());
9239 g.asks += 1;
9240 self.state.github_text = Some(g.clone());
9241 self.state.event(
9242 "github-text",
9243 format!("asking the owner about the withheld text ({})", q.short()),
9244 );
9245 self.state.save()?;
9246 store
9247 .put_executable(&mut q)
9248 .context("file the posting-gate question")?;
9249 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9250 tracing::warn!(
9251 "could not notify about posting-gate question {}: {e:#}",
9252 q.short()
9253 );
9254 }
9255 self.park_on_github_text(&g)?;
9256 return Ok(None);
9257 }
9258 }
9259 let resolved = g.chosen_title.clone();
9260 let original = g.choice == Some(GithubTextChoice::Original);
9261 self.state.event(
9263 "github-text",
9264 if original {
9265 format!(
9266 "posting the original text for: {}",
9267 match (g.title, g.body) {
9268 (true, true) => "title, description",
9269 (true, false) => "title",
9270 _ => "description",
9271 }
9272 )
9273 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9274 "posting the saved rewrite title and description".to_owned()
9275 } else if resolved.is_some() {
9276 "posting the owner's replacement title".to_owned()
9277 } else {
9278 "posting the neutral text for the withheld fields".to_owned()
9279 },
9280 );
9281 self.state.github_text = Some(g.clone());
9282 self.state.save()?;
9283 if g.choice == Some(GithubTextChoice::Rewrite)
9284 && let Some(r) = g.rewrite.as_ref()
9285 {
9286 return Ok(Some(PrMessage {
9287 title: r.title.clone(),
9288 body: r.body.clone(),
9289 }));
9290 }
9291 let w = gt::Withheld {
9292 title: g.title,
9293 body: g.body,
9294 categories: g.categories.clone(),
9295 };
9296 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9297 if let Some(t) = resolved {
9298 title = t;
9299 }
9300 let english = gt::LanguageDecision {
9302 title_english: true,
9303 body_english: true,
9304 body_complete: true,
9305 };
9306 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9307 if original {
9308 if g.title {
9311 title = pr.title.clone();
9312 }
9313 if g.body {
9314 body = pr.body.clone();
9315 }
9316 }
9317 Ok(Some(PrMessage { title, body }))
9318 }
9319
9320 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9321 self.state.github_text = Some(g.clone());
9322 self.state.end_pass_parked(ParkCause::Question);
9323 self.state.parked = true;
9324 self.state.event(
9325 "github-text",
9326 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9327 );
9328 self.state.save()
9329 }
9330
9331 async fn judge_pr_language(
9334 &mut self,
9335 cwd: &Path,
9336 pr: &PrMessage,
9337 ) -> Option<crate::github_text::LanguageDecision> {
9338 self.state.config.roles.language_judge.as_ref()?;
9339 if !self.state.config.graph.language_gate() {
9340 return None;
9341 }
9342 let decision =
9343 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9344 self.state.event(
9345 "github-text",
9346 if decision.is_some() {
9347 "language decided by the language judge"
9348 } else {
9349 "language judge unavailable; using built-in heuristics"
9350 },
9351 );
9352 decision
9353 }
9354
9355 async fn merge(&mut self) -> Result<()> {
9356 if self.resource_halt()? || self.hold_base_sync().await? {
9357 return Ok(());
9358 }
9359 if self
9374 .state
9375 .base_sync
9376 .as_ref()
9377 .is_some_and(|s| s.conflict.is_some())
9378 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9379 != Some(RunStatus::Gating)
9380 || !self.state.gate_allows_progress()
9388 {
9389 return Ok(());
9390 }
9391 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9400 return Ok(());
9401 }
9402 let Some(winner) = self.state.winner().cloned() else {
9403 return Ok(());
9404 };
9405 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9406 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9407 self.state.status = RunStatus::Blocked;
9408 self.state.event(
9409 "gate",
9410 "winner changed after gate; fresh verification required",
9411 );
9412 self.state.save()?;
9413 return Ok(());
9414 }
9415 let wait = self.repository_wait();
9416 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9417 return Ok(());
9418 }
9419 let repo = self.state.repo.clone();
9420 let base = self.state.base_branch.clone();
9421 let mode = self.state.config.merge.mode;
9422 let style = self.state.config.merge.style;
9423 let facts = if is_review_run(&self.state) {
9424 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9425 let start = review_base(
9426 &repo,
9427 &self.state.config.merge.remote,
9428 &base,
9429 &self.state.base_commit,
9430 &winner.branch,
9431 )
9432 .await;
9433 branch_facts(&repo, &start, &winner.branch).await
9434 } else {
9435 None
9436 };
9437 let leaked = if is_review_run(&self.state) {
9440 leaked_subjects(&self.state, &winner.branch).await
9441 } else {
9442 Some(Vec::new())
9443 };
9444 let Some(pr) = self
9448 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9449 .await?
9450 else {
9451 return Ok(());
9452 };
9453 let message = pr.commit_message();
9454
9455 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9456 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9457 return Ok(());
9458 }
9459 let outcome = match mode {
9460 MergeMode::None => MergeOutcome {
9461 mode,
9462 ok: true,
9463 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9464 empty: false,
9465 },
9466 MergeMode::Pr | MergeMode::Local
9467 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9468 {
9469 MergeOutcome {
9470 mode,
9471 ok: false,
9472 detail: empty_candidate_detail(&self.state, &base),
9473 empty: true,
9474 }
9475 }
9476 MergeMode::Local => {
9477 let on = git::current_branch(&repo).await?;
9478 if on.as_deref() != Some(base.as_str()) {
9479 MergeOutcome {
9480 mode,
9481 ok: false,
9482 detail: format!(
9483 "{} has {} checked out, not the base branch {base}",
9484 repo.display(),
9485 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9486 ),
9487 empty: false,
9488 }
9489 } else if !git::is_clean(&repo).await? {
9490 MergeOutcome {
9491 mode,
9492 ok: false,
9493 detail: format!("{} is dirty; refusing to merge", repo.display()),
9494 empty: false,
9495 }
9496 } else {
9497 let out = match style {
9498 MergeStyle::Merge => {
9499 git::merge_no_ff(&repo, &winner.branch, &message).await?
9500 }
9501 MergeStyle::Squash => {
9502 git::merge_squash(&repo, &winner.branch, &message).await?
9503 }
9504 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9505 };
9506 MergeOutcome {
9507 mode,
9508 ok: out.ok(),
9509 detail: if out.ok() { out.stdout } else { out.stderr },
9510 empty: false,
9511 }
9512 }
9513 }
9514 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9515 self.state.pr_publication_pending = true;
9516 self.state.save()?;
9517 let out =
9518 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9519 self.state.pr_publication_pending = out.is_err();
9520 MergeOutcome {
9521 mode,
9522 ok: out.is_ok(),
9523 detail: match out {
9524 Ok(url) => url,
9525 Err(e) => format!("{e:#}"),
9526 },
9527 empty: false,
9528 }
9529 }
9530 MergeMode::Pr => {
9531 let remote = self.state.config.merge.remote.clone();
9532 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9533 let pushed = if self.state.foreign_checkout.is_some() {
9534 let refspec = format!("{delivery_head}:refs/heads/{public}");
9535 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9536 } else {
9537 git::push(&winner.worktree, &remote, &winner.branch).await?
9538 };
9539 if !pushed.ok() {
9540 MergeOutcome {
9541 mode,
9542 ok: false,
9543 detail: pushed.stderr,
9544 empty: false,
9545 }
9546 } else {
9547 if self.state.foreign_checkout.is_some() {
9548 self.state.pr_pushed_head = Some(delivery_head.clone());
9549 }
9550 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9556 let out = match pr_merge_plan(found) {
9557 PrPlan::Create => {
9558 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9559 .await
9560 }
9561 PrPlan::Adopt { url, title } => {
9562 self.state
9563 .event("merge", format!("Pr: adopted open pull request {url}"));
9564 if self.state.github_text.as_ref().is_some_and(|g| {
9565 g.resolved
9566 && g.choice == Some(GithubTextChoice::Rewrite)
9567 && g.rewrite.is_some()
9568 }) {
9569 land::set_pr_text_verbatim(
9572 &winner.worktree,
9573 &url,
9574 &pr.title,
9575 &pr.body,
9576 )
9577 .await
9578 .map(|()| url)
9579 } else {
9580 if title != pr.title
9581 && (!is_review_run(&self.state)
9582 || leaked
9583 .as_deref()
9584 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9585 && let Err(e) =
9586 if self.state.github_text.as_ref().is_some_and(|g| {
9587 g.resolved
9588 && g.title
9589 && g.choice == Some(GithubTextChoice::Original)
9590 }) {
9591 land::set_pr_title_verbatim(
9593 &winner.worktree,
9594 &url,
9595 &pr.title,
9596 )
9597 .await
9598 } else {
9599 land::set_pr_title(
9600 &mut self.state,
9601 &winner.worktree,
9602 &url,
9603 &pr.title,
9604 )
9605 .await
9606 }
9607 {
9608 tracing::warn!("could not refresh title of {url}: {e:#}");
9609 self.state
9610 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9611 }
9612 Ok(url)
9613 }
9614 }
9615 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9616 };
9617 match out {
9618 Ok(url) => MergeOutcome {
9619 mode,
9620 ok: true,
9621 detail: url,
9622 empty: false,
9623 },
9624 Err(e) => MergeOutcome {
9625 mode,
9626 ok: false,
9627 detail: e.to_string(),
9628 empty: false,
9629 },
9630 }
9631 }
9632 }
9633 };
9634
9635 self.state.status = match (mode, outcome.ok) {
9636 (MergeMode::None, _) => RunStatus::Ready,
9637 (_, true) if outcome.empty => RunStatus::Ready,
9638 (MergeMode::Pr, true) => RunStatus::Ready,
9639 (_, true) => RunStatus::Merged,
9640 (_, false) => RunStatus::Blocked,
9641 };
9642 self.state.event(
9643 "merge",
9644 format!(
9645 "{:?}: {}",
9646 mode,
9647 outcome.detail.lines().next().unwrap_or("")
9648 ),
9649 );
9650 self.state.merge = Some(outcome);
9651 self.state.save()?;
9652
9653 if self.state.config.graph.land
9659 && mode == MergeMode::Pr
9660 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9661 {
9662 self.run_land().await?;
9663 }
9664 self.settle_questions();
9669 Ok(())
9670 }
9671
9672 async fn run_land(&mut self) -> Result<()> {
9683 crate::pr_head::ensure_owner(&self.state)?;
9684 let url = self
9685 .state
9686 .pr
9687 .as_ref()
9688 .map(|pr| pr.url.clone())
9689 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9690 .unwrap_or_default();
9691 let url = url.lines().next().unwrap_or("").trim().to_owned();
9692 if !url.starts_with("http") {
9693 return Ok(());
9694 }
9695 loop {
9698 match land::land(&mut self.state, &url).await {
9699 Ok(_)
9700 if self
9701 .state
9702 .merge_repairs
9703 .last()
9704 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9705 {
9706 if self.repair_merge_review().await? {
9707 continue;
9708 }
9709 }
9710 Ok(_) if self.state.resource_block.is_some() => {
9711 return self.stop_for_resource();
9712 }
9713 Ok(pr) if self.state.parked => {
9714 let _ = pr;
9718 }
9719 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9720 self.state.status = RunStatus::Superseded;
9721 self.state.save()?;
9722 }
9723 Ok(pr) => {
9724 self.state.status = match pr.state {
9725 land::PrLifecycle::Merged => RunStatus::Merged,
9726 _ => RunStatus::Blocked,
9727 };
9728 if bump::should_release_bump(self.state.status)
9735 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9736 {
9737 self.state
9743 .event("bump", format!("release bump skipped: {e:#}"));
9744 }
9745 if self.state.status == RunStatus::Merged {
9748 crate::followup::after_merge(&mut self.state, &pr.url).await;
9749 }
9750 self.state.save()?;
9751 }
9752 Err(e) => {
9753 if e.is::<crate::repo_lock::ResourceBlocked>() {
9754 return Err(e);
9755 }
9756 self.state.status = RunStatus::Blocked;
9757 self.state.event("land", format!("gave up: {e}"));
9758 self.state.save()?;
9759 }
9760 }
9761 break;
9762 }
9763 Ok(())
9764 }
9765
9766 async fn repair_merge_review(&mut self) -> Result<bool> {
9769 use crate::run::MergeRepairPhase as Phase;
9770 crate::pr_head::ensure_owner(&self.state)?;
9771 let i = self.state.merge_repairs.len() - 1;
9772 let request = self.state.merge_repairs[i].clone();
9773 if request.phase == Phase::Fixing {
9774 let restored = match self.state.winner().cloned() {
9775 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9776 None => Err(anyhow::anyhow!("repair has no winning branch")),
9777 };
9778 let why = match restored {
9779 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9780 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9781 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9782 };
9783 self.hold_merge_repair(i, why)?;
9784 return Ok(false);
9785 }
9786 if request.phase == Phase::Reviewing && self.state.gate_ran {
9791 let winner = self
9792 .state
9793 .winner()
9794 .context("repair has no winning branch")?;
9795 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9796 if self.state.gate_head.as_deref() != Some(head.as_str())
9797 || !git::is_clean(&winner.worktree).await?
9798 {
9799 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9800 return Ok(false);
9801 }
9802 }
9803 let failure = if i >= self.state.config.graph.land_rounds
9804 || self.state.config.graph.review_rounds == 0
9805 {
9806 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9807 } else {
9808 None
9809 };
9810 if let Some(why) = failure {
9811 self.hold_merge_repair(i, why)?;
9812 return Ok(false);
9813 }
9814 if request.phase == Phase::Requested {
9815 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9816 Ok(pr) => pr,
9817 Err(e) => {
9818 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9819 return Ok(false);
9820 }
9821 };
9822 self.state.merge_repairs[i].phase = Phase::Fixing;
9823 self.state.land_approval = None;
9824 self.state.land_armed_head = None;
9825 self.state.save()?;
9826 let evidence = serde_json::to_string_pretty(&request)?;
9829 let reason = format!(
9830 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9831 );
9832 let budget = self.state.config.graph.land_rounds;
9833 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9834 match result {
9835 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9836 Ok(land::Fixed::Deferred) => {
9837 let worktree = self
9841 .state
9842 .winner()
9843 .context("repair has no winning branch")?
9844 .worktree
9845 .clone();
9846 self.graft_restore(&worktree, &request.head).await?;
9847 self.state.merge_repairs[i].phase = Phase::Requested;
9848 self.stop_for_resource()?;
9849 return Ok(false);
9850 }
9851 Ok(land::Fixed::Failed(why)) => {
9852 self.hold_merge_repair(
9853 i,
9854 format!(
9855 "repair failed: {why}; inspect the branch and run an explicit review"
9856 ),
9857 )?;
9858 return Ok(false);
9859 }
9860 Err(e) => {
9861 self.hold_merge_repair(
9862 i,
9863 format!(
9864 "repair failed: {e:#}; inspect the branch and run an explicit review"
9865 ),
9866 )?;
9867 return Ok(false);
9868 }
9869 }
9870 self.begin_merge_repair_review(i)?;
9874 }
9875 self.review_loop().await?;
9876 self.gate().await?;
9877 if self.resource_halt()? || self.state.parked {
9878 return Ok(false);
9879 }
9880 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9881 != Some(RunStatus::Gating)
9882 || !self.state.gate_allows_progress()
9883 {
9884 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9885 return Ok(false);
9886 }
9887 let winner = self
9890 .state
9891 .winner()
9892 .cloned()
9893 .context("repair has no winning branch")?;
9894 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9895 if self.state.gate_head.as_deref() != Some(after.as_str())
9896 || !git::is_clean(&winner.worktree).await?
9897 {
9898 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9899 return Ok(false);
9900 }
9901 let expected = self
9902 .state
9903 .pr_pushed_head
9904 .as_deref()
9905 .unwrap_or(&request.head)
9906 .to_owned();
9907 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9908 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9911 return Ok(false);
9912 }
9913 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9914 let refspec = format!("{after}:refs/heads/{public}");
9915 let push = git::git_raw(
9916 &winner.worktree,
9917 &["push", &lease, &self.state.config.merge.remote, &refspec],
9918 )
9919 .await;
9920 match push {
9921 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9922 Ok(out) => {
9923 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9924 return Ok(false);
9925 }
9926 Err(e) => {
9927 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9928 return Ok(false);
9929 }
9930 }
9931 self.state.merge_repairs[i].phase = Phase::Complete;
9932 self.state.status = RunStatus::Landing;
9933 self.state.event(
9934 "repair",
9935 "new review and gate completed; fresh owner merge decision required",
9936 );
9937 self.state.save()?;
9938 Ok(true)
9939 }
9940
9941 fn merge_repair_artifacts(&self) -> PathBuf {
9942 let artifacts = agent::artifacts_dir(&self.state.dir());
9943 match self
9944 .state
9945 .merge_repairs
9946 .last()
9947 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9948 {
9949 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9950 None => artifacts,
9951 }
9952 }
9953
9954 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9955 if let Some(e2e) = self.state.head_e2e.take() {
9956 self.state
9957 .verification_history
9958 .push(crate::run::VerificationEvidence {
9959 superseded_by: None,
9960 archived_at: Some(Timestamp::now()),
9961 fixes_spent: Some(self.state.gate_fixes.len()),
9962 gate_head: None,
9963 gate_ran: false,
9964 gate: Vec::new(),
9965 e2e: Some(e2e),
9966 });
9967 }
9968 self.state.reviews.clear();
9969 self.state.gate.clear();
9970 self.state.gate_ran = false;
9971 self.state.gate_head = None;
9972 self.state.gate_fixes.clear();
9973 self.state.gate_repair_e2e = None;
9974 self.state.pre_gate.clear();
9975 self.state.pre_gate_commit = None;
9976 self.state.contested_handoff = None;
9977 self.state.land_reruns = None;
9978 self.state.land_approval = None;
9979 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9980 self.state.status = RunStatus::Reviewing;
9981 self.state.save()
9982 }
9983
9984 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9985 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9986 self.state.merge_repairs[index].failure = Some(why.clone());
9987 self.state.status = RunStatus::Blocked;
9988 self.state.event("repair", why);
9989 self.state.save()
9990 }
9991
9992 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9996 if let Some(existing) = self.state.seats.get(key)
9997 && existing.agent == agent
9998 {
9999 return existing.clone();
10000 }
10001 let fresh = if self.state.seats.contains_key(key) {
10004 handover_seat(key, agent, self.state.next_seat_seed())
10005 } else {
10006 SeatState::new(key, agent, self.state.seed)
10007 };
10008 self.state.seats.insert(key.to_owned(), fresh.clone());
10009 fresh
10010 }
10011
10012 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10018 match self.state.seats.get(key) {
10019 Some(s) if s.agent != spec.id => {
10020 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10021 }
10022 _ => spec,
10023 }
10024 }
10025
10026 fn view(&self, c: &Candidate) -> CandidateView {
10028 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10029 .unwrap_or_default();
10030 let (patch, _) = blind::sanitize_patch(
10031 &format!("candidate {} patch", c.label),
10032 &raw,
10033 &self.state.config.blind,
10034 );
10035 CandidateView {
10036 label: c.label,
10037 branch: c.branch.clone(),
10038 summary: c.summary.clone(),
10039 stat: c.stat.clone(),
10040 patch,
10041 }
10042 }
10043
10044 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10046 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10047 prompt::judge(
10048 "(see above)",
10049 &views,
10050 self.roles.judges.len(),
10051 base_short,
10052 "en",
10053 )
10054 }
10055
10056 fn vote_prompt_full(
10062 &self,
10063 j: usize,
10064 labels: &[char],
10065 language: &str,
10066 candidates: &[Candidate],
10067 base_short: &str,
10068 ) -> String {
10069 let mut text = format!(
10070 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10071 prompt::final_vote(labels, language),
10072 self.state.instruction,
10073 self.candidate_block(candidates, base_short)
10074 );
10075 if let Some(own) = self
10076 .state
10077 .judgements
10078 .get(j)
10079 .filter(|r| !r.ranking.is_empty())
10080 {
10081 let reasons = own
10082 .reasons
10083 .iter()
10084 .map(|(k, v)| format!("- {k}: {v}"))
10085 .collect::<Vec<_>>()
10086 .join("\n");
10087 text.push_str(&format!(
10088 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10089 own.ranking.iter().collect::<String>(),
10090 if reasons.is_empty() {
10091 ""
10092 } else {
10093 ", because:\n"
10094 }
10095 ));
10096 }
10097 if !self.state.deliberation.is_empty() {
10098 text.push_str("\n# What was argued before this vote\n");
10099 for t in self.transcript(&[], j) {
10100 text.push_str(&format!(
10101 "\n## {}{}\n\n{}\n",
10102 t.who,
10103 if t.is_self { " (you)" } else { "" },
10104 t.body.trim()
10105 ));
10106 }
10107 }
10108 text
10109 }
10110
10111 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10118 let mut turns = Vec::new();
10119 for j in &self.state.judgements {
10120 if j.ranking.is_empty() {
10121 continue;
10122 }
10123 let reasons = j
10124 .reasons
10125 .iter()
10126 .map(|(k, v)| format!("- {k}: {v}"))
10127 .collect::<Vec<_>>()
10128 .join("\n");
10129 turns.push(Turn {
10130 who: format!("Judge {} (opening ranking)", j.judge),
10131 is_self: j.judge == self_idx + 1,
10132 body: format!(
10133 "Ranked {}{}{reasons}",
10134 j.ranking.iter().collect::<String>(),
10135 if reasons.is_empty() {
10136 ""
10137 } else {
10138 ", because:\n"
10139 }
10140 ),
10141 });
10142 }
10143 for t in self
10144 .state
10145 .deliberation
10146 .iter()
10147 .flat_map(|r| r.turns.iter())
10148 .chain(current)
10149 {
10150 turns.push(Turn {
10151 who: format!("Judge {}", t.judge),
10152 is_self: t.judge == self_idx + 1,
10153 body: t.body.clone(),
10154 });
10155 }
10156 turns
10157 }
10158}
10159
10160fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10162 agent::has_session(spec.kind, seat, sessions)
10163}
10164
10165fn next_untried_in_roster<'a>(
10186 roster: &'a [AgentSpec],
10187 start: usize,
10188 tried: &BTreeSet<String>,
10189) -> Option<&'a AgentSpec> {
10190 roster
10191 .get(start + 1..)?
10192 .iter()
10193 .find(|s| !tried.contains(&s.id))
10194}
10195
10196fn pick_successor<'a>(
10217 roster: &'a [AgentSpec],
10218 start: usize,
10219 tried: &BTreeSet<String>,
10220 carried: Option<&BTreeSet<String>>,
10221 others: &BTreeSet<String>,
10222) -> Option<&'a AgentSpec> {
10223 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10224 roster
10225 .get(start + 1..)
10226 .and_then(|tail| {
10227 tail.iter()
10228 .filter(free)
10229 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10230 })
10231 .or_else(|| {
10232 carried
10233 .is_some()
10234 .then(|| roster.iter().find(free))
10235 .flatten()
10236 })
10237 .or_else(|| match carried {
10238 Some(c) => next_for_seat(roster, start, tried, c),
10239 None => next_untried_in_roster(roster, start, tried),
10240 })
10241}
10242
10243fn next_for_seat<'a>(
10253 roster: &'a [AgentSpec],
10254 start: usize,
10255 round_tried: &BTreeSet<String>,
10256 carried_failed: &BTreeSet<String>,
10257) -> Option<&'a AgentSpec> {
10258 roster
10259 .get(start + 1..)?
10260 .iter()
10261 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10262 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10263}
10264
10265fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10271 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10272 return spec;
10273 };
10274 let ok = |id: &str| !h.failed.contains(id);
10275 if let Some(last) = h.last_ok.as_deref()
10276 && ok(last)
10277 && let Some(s) = roster.iter().find(|s| s.id == last)
10278 {
10279 return s.clone();
10280 }
10281 if ok(&spec.id) {
10282 return spec;
10283 }
10284 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10285 roster
10286 .get(from + 1..)
10287 .into_iter()
10288 .flatten()
10289 .chain(roster.iter())
10290 .find(|s| ok(&s.id))
10291 .cloned()
10292 .unwrap_or(spec)
10293}
10294
10295pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10299 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10300}
10301
10302const TIMED_OUT: &str = "timed out";
10305
10306impl FailClass {
10307 fn of(out: &AgentOutcome) -> Option<Self> {
10309 match out {
10310 AgentOutcome::Ok(_) => None,
10311 AgentOutcome::Quota(_) => Some(Self::Quota),
10312 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10313 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10314 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10315 AgentOutcome::ResourceBlocked(_) => None,
10318 }
10319 }
10320
10321 fn stem_word(&self) -> &'static str {
10323 match self {
10324 Self::Quota => "quota",
10325 _ => "handover",
10326 }
10327 }
10328}
10329
10330fn failure_signature(msg: &str) -> String {
10334 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10335 if let Some(code) = line
10338 .strip_prefix("process exited with no usable output (exit code ")
10339 .and_then(|s| s.strip_suffix(')'))
10340 .and_then(|s| s.parse::<i32>().ok())
10341 {
10342 line = format!("exited with some({code}) and no usable output");
10343 } else if matches!(
10344 line.as_str(),
10345 "process exited with no usable output (no exit code reported)"
10346 | "process exited with no usable output (exit code unavailable)"
10347 ) {
10348 line = "exited with none and no usable output".to_owned();
10349 }
10350 let mut out = Vec::new();
10351 for word in line.split_whitespace() {
10352 if word.contains('/') || word.contains('\\') {
10353 out.push("<path>".to_owned());
10354 continue;
10355 }
10356 let mut w = String::new();
10357 let mut in_digits = false;
10358 for c in word.chars() {
10359 if c.is_ascii_digit() {
10360 if !in_digits {
10361 w.push('#');
10362 }
10363 in_digits = true;
10364 } else {
10365 in_digits = false;
10366 w.push(c);
10367 }
10368 }
10369 out.push(w);
10370 }
10371 out.join(" ").chars().take(120).collect()
10372}
10373
10374fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10381 match cur {
10382 FailClass::Quota | FailClass::Timeout => true,
10383 FailClass::Other(_) => prev != Some(cur),
10384 }
10385}
10386
10387fn fail_reason(out: &AgentOutcome) -> String {
10389 match out {
10390 AgentOutcome::Ok(_) => String::new(),
10391 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10392 AgentOutcome::Dropped(o) => format!(
10393 "the CLI dropped the stream ({})",
10394 o.dropped
10395 .as_ref()
10396 .map(|d| d.why.as_str())
10397 .unwrap_or("it ended without delivering its answer")
10398 ),
10399 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10400 e.lines().next().unwrap_or("").chars().take(160).collect()
10401 }
10402 }
10403}
10404
10405pub(crate) fn record_handover(
10408 state: &mut RunState,
10409 node: &str,
10410 seat: &str,
10411 from: &str,
10412 to: &str,
10413 class: &FailClass,
10414 reason: &str,
10415) {
10416 let message = if *class == FailClass::Quota {
10417 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10418 } else {
10419 format!("{seat}: handed over {from} -> {to} ({reason})")
10420 };
10421 state.event(node, message);
10422 state.handovers.push(Handover {
10423 at: Timestamp::now(),
10424 node: node.to_owned(),
10425 seat: seat.to_owned(),
10426 from: from.to_owned(),
10427 to: to.to_owned(),
10428 reason: reason.to_owned(),
10429 });
10430}
10431
10432fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10446 commands.iter().any(|c| c.exit_code.is_none())
10447}
10448
10449fn verified_noop_claim(
10462 usable: bool,
10463 commands: &[agent::CommandEvidence],
10464 text: &str,
10465) -> Option<String> {
10466 (usable && !has_unconfirmed_command(commands))
10467 .then(|| verdict::verified_noop(text))
10468 .flatten()
10469}
10470
10471fn short(commit: &str) -> String {
10472 commit.chars().take(7).collect()
10473}
10474
10475fn make_executable(path: &Path) -> Result<()> {
10476 #[cfg(unix)]
10477 {
10478 use std::os::unix::fs::PermissionsExt as _;
10479 let mut perms = std::fs::metadata(path)?.permissions();
10480 perms.set_mode(0o755);
10481 std::fs::set_permissions(path, perms)?;
10482 }
10483 #[cfg(not(unix))]
10484 {
10485 let _ = path;
10486 }
10487 Ok(())
10488}
10489
10490struct WaveCtx<'a> {
10497 run: &'a str,
10500 node: &'a str,
10502 prompts: &'a Prompts,
10503 cache: Option<&'a Path>,
10505 round: Option<usize>,
10508 carry_seats: bool,
10512 pauses: Option<(&'a Pause, &'a Pause)>,
10515}
10516
10517fn repository_wait(
10519 state: &RunState,
10520 pauses: Option<(&Pause, &Pause)>,
10521) -> crate::repo_lock::WaitPolicy {
10522 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10523 let inherited = crate::repo_lock::WaitPolicy::current();
10524 crate::repo_lock::WaitPolicy::new(
10525 Duration::from_secs(state.config.graph.timeout_resource_wait),
10526 move || {
10527 inherited.cancelled()
10528 || pauses
10529 .as_ref()
10530 .is_some_and(|(a, b)| a.parked() || b.parked())
10531 },
10532 )
10533}
10534
10535fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10536 if error.is::<crate::repo_lock::ResourceBlocked>() {
10537 state.resource_block = Some(crate::run::ResourceBlock {
10538 cause: crate::run::ResourceBlockCause::Other,
10539 node: "repository".into(),
10540 kind: if error.to_string().contains("disk pressure") {
10541 "disk pressure"
10542 } else if error.to_string().contains("disk admission") {
10543 "disk admission"
10544 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10545 "incremental cache admission"
10546 } else {
10547 "repository coordination"
10548 }
10549 .into(),
10550 detail: error.to_string(),
10551 at: Timestamp::now(),
10552 });
10553 state.event("resource", error.to_string());
10554 }
10555}
10556
10557async fn coordinate_operation(
10558 state: &mut RunState,
10559 cwd: &Path,
10560 wait: &crate::repo_lock::WaitPolicy,
10561) -> Result<Option<crate::repo_lock::SeatLease>> {
10562 let result =
10563 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10564 .await;
10565 if let Err(e) = &result {
10566 record_repository_block(state, e);
10567 }
10568 result
10569}
10570
10571async fn run_one(
10573 job: SeatJob,
10574 sem: Arc<Semaphore>,
10575 ctx: &WaveCtx<'_>,
10576 state: &mut RunState,
10577 attempt: usize,
10578) -> (SeatState, AgentOutcome) {
10579 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10580 .await
10581 .pop()
10582 .expect("one job in, one result out");
10583 (seat, out)
10584}
10585
10586async fn wave(
10592 jobs: Vec<SeatJob>,
10593 sem: Arc<Semaphore>,
10594 ctx: &WaveCtx<'_>,
10595 state: &mut RunState,
10596 attempt: usize,
10597) -> Vec<(usize, SeatState, AgentOutcome)> {
10598 let WaveCtx {
10599 run,
10600 node,
10601 prompts,
10602 cache,
10603 round,
10604 carry_seats: _,
10605 pauses,
10606 } = *ctx;
10607 for job in &jobs {
10608 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10609 }
10610 if let Err(e) = state.save() {
10611 tracing::warn!("could not persist in-progress seats: {e:#}");
10616 }
10617 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10633 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10637 let cache_guard = if let Some(cache_dir) = cache {
10638 if jobs_had_a_writer {
10639 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10640 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10641 match acquire_cache_lease(
10642 state,
10643 cache_dir,
10644 &owner,
10645 resource_limit,
10646 node,
10647 node,
10648 &cancel,
10649 )
10650 .await
10651 {
10652 Ok(Some(g)) => Some(g),
10653 Ok(None) => {
10656 if let Some(block) = &mut state.resource_block {
10657 block.cause = crate::run::ResourceBlockCause::Cancelled {
10658 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10659 };
10660 }
10661 return block_wave(
10662 jobs,
10663 state,
10664 node,
10665 attempt,
10666 "parked while waiting for the shared build cache",
10667 );
10668 }
10669 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10672 Err(e) => {
10673 let why = format!("{e:#}");
10677 state.event(
10678 "resource",
10679 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10680 );
10681 return block_wave(jobs, state, node, attempt, &why);
10682 }
10683 }
10684 } else {
10685 None
10686 }
10687 } else {
10688 None
10689 };
10690 state.seats_spawning(node, attempt);
10693 if let Err(e) = state.save() {
10694 tracing::warn!("could not persist the start of a wave: {e:#}");
10695 }
10696 let slot_limit = resource_limit
10700 + jobs
10701 .iter()
10702 .map(|j| j.timeout)
10703 .max()
10704 .unwrap_or_default()
10705 .saturating_mul(jobs.len() as u32);
10706 let mut set = tokio::task::JoinSet::new();
10707 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10710 let overlay = prompts.overlay(node);
10711 let repository_wait = repository_wait(state, pauses);
10712 for (i, mut job) in jobs.into_iter().enumerate() {
10713 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10714 if cache.is_some() {
10715 job.prompt.push('\n');
10716 job.prompt
10717 .push_str(&prompt::build_cache_note(node, job.allow_write));
10718 }
10719 let sem = Arc::clone(&sem);
10720 let run = run.to_owned();
10721 let node = node.to_owned();
10722 let attachments = if node == "implement" {
10725 state.attachments.clone()
10726 } else {
10727 Vec::new()
10728 };
10729 let cache = cache
10740 .filter(|_| job.allow_write && cache_guard.is_some())
10741 .map(Path::to_path_buf);
10742 let spawned_tx = spawned_tx.clone();
10743 let repository_wait = repository_wait.clone();
10744 let coordination = state.config.coordination.clone();
10745 let fetched_base = state.fetched_base.clone();
10746 set.spawn(async move {
10747 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10751 Ok(p) => p,
10752 Err(_) => {
10753 return (
10754 i,
10755 job.seat,
10756 AgentOutcome::ResourceBlocked(format!(
10757 "no parallel slot became free within {}s",
10758 slot_limit.as_secs()
10759 )),
10760 false,
10761 );
10762 }
10763 };
10764 spawned_tx.send(job.seat.key.clone()).ok();
10765 let mut seat = job.seat;
10766 let out = repository_wait
10767 .scope(agent::invoke(
10768 &job.spec,
10769 &mut seat,
10770 &Invocation {
10771 coordination: coordination.as_ref(),
10772 fetched_base: fetched_base.as_ref(),
10773 cwd: &job.cwd,
10774 prompt: &job.prompt,
10775 timeout: job.timeout,
10776 allow_write: job.allow_write,
10777 sessions: job.sessions,
10778 artifacts: &job.artifacts,
10779 stem: &job.stem,
10780 run: &run,
10781 node: &node,
10782 cache_dir: cache.as_deref(),
10783 attachments: &attachments,
10784 writable: &[],
10785 },
10786 ))
10787 .await;
10788 let interrupted = out
10789 .as_ref()
10790 .err()
10791 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10792 let out = match out {
10793 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10794 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10795 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10803 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10804 Ok(o) => AgentOutcome::Failed(format!(
10805 "process exited with no usable output ({})",
10806 crate::notices::exit_status(o.exit_code)
10807 )),
10808 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10809 AgentOutcome::ResourceBlocked(e.to_string())
10810 }
10811 Err(e) => AgentOutcome::Failed(e.to_string()),
10812 };
10813 (i, seat, out, interrupted)
10814 });
10815 }
10816 drop(spawned_tx);
10817 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10818 loop {
10819 let joined = tokio::select! {
10820 Some(key) = spawned_rx.recv() => {
10821 state.seat_spawned(&key);
10823 if let Err(e) = state.save() {
10824 tracing::warn!("could not persist a seat's start: {e:#}");
10825 }
10826 continue;
10827 }
10828 joined = set.join_next() => joined,
10829 };
10830 let Some(joined) = joined else {
10831 break;
10832 };
10833 let (i, seat, out, interrupted) = match joined {
10834 Ok(v) => v,
10835 Err(e) => {
10839 tracing::error!("agent task panicked: {e}");
10840 continue;
10841 }
10842 };
10843 state.seat_finished(&seat.key);
10844 if interrupted
10845 && node == "implement"
10846 && let Some(label) = state
10847 .candidates
10848 .iter()
10849 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10850 .map(|candidate| candidate.label)
10851 {
10852 state.candidate_preservation.insert(
10853 label,
10854 "interrupted implementation awaiting persistence".into(),
10855 );
10856 state.interrupted_implementations.insert(label);
10857 }
10858 if let AgentOutcome::ResourceBlocked(why) = &out
10859 && state.resource_block.is_none()
10860 {
10861 state.resource_block = Some(crate::run::ResourceBlock {
10862 cause: crate::run::ResourceBlockCause::Other,
10863 node: node.to_owned(),
10864 kind: if why.contains("disk pressure") {
10865 "disk pressure"
10866 } else if why.contains("disk admission") {
10867 "disk admission"
10868 } else if why.contains(crate::incremental::BLOCKED) {
10869 "incremental cache admission"
10870 } else if why.contains("repository coordination") {
10871 "repository coordination"
10872 } else {
10873 "parallel slot"
10874 }
10875 .to_owned(),
10876 detail: why.clone(),
10877 at: Timestamp::now(),
10878 });
10879 let activity = if interrupted {
10880 "agent execution interrupted"
10881 } else {
10882 "no agent was started"
10883 };
10884 state.event("resource", format!("{node}: {activity} - {why}"));
10885 }
10886 record_jobs(state, node, round, &seat.key, &out);
10887 if let Err(e) = state.save() {
10888 tracing::warn!("could not persist a seat's completion: {e:#}");
10889 }
10890 if collected.len() <= i {
10891 collected.resize_with(i + 1, || None);
10892 }
10893 collected[i] = Some((i, seat, out));
10894 }
10895 if state
10901 .active
10902 .values()
10903 .any(|a| a.node == node && a.attempt == attempt)
10904 {
10905 state
10906 .active
10907 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10908 if let Err(e) = state.save() {
10909 tracing::warn!("could not persist the end of a wave: {e:#}");
10910 }
10911 }
10912 if let Some(cache_dir) = cache
10919 && jobs_had_a_writer
10920 {
10921 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10922 }
10923 if let Some(guard) = cache_guard {
10924 guard.release();
10925 }
10926 collected.into_iter().flatten().collect()
10927}
10928
10929fn record_jobs(
10940 state: &mut RunState,
10941 node: &str,
10942 round: Option<usize>,
10943 seat: &str,
10944 out: &AgentOutcome,
10945) {
10946 let commands: &[agent::CommandEvidence] = match out {
10947 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10948 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10949 };
10950 let checked_at = Timestamp::now();
10951 for c in commands {
10952 state.jobs.push(JobRecord {
10953 node: node.to_owned(),
10954 round,
10955 seat: seat.to_owned(),
10956 id: c.id.clone(),
10957 description: c.description.clone(),
10958 checked_at,
10959 status: match c.exit_code {
10960 Some(0) => JobStatus::Completed,
10961 Some(_) => JobStatus::Failed,
10962 None => JobStatus::Unknown,
10963 },
10964 exit_code: c.exit_code,
10965 result_summary: c.result_summary.clone(),
10966 source: c.source.clone(),
10967 });
10968 }
10969}
10970
10971fn round_is_clean(
10992 blocking: usize,
10993 e2e_ok: bool,
10994 answered: usize,
10995 expected: usize,
10996 quota_missing: usize,
10997 policy: IncompleteReviewPolicy,
10998) -> bool {
10999 if blocking != 0 || !e2e_ok {
11000 return false;
11001 }
11002 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11003 return true;
11004 }
11005 answered > 0 && expected - answered <= quota_missing
11006}
11007
11008pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11032 if max_rounds == 0
11033 || reviews.last().is_some_and(|r| {
11034 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11035 })
11036 {
11037 return Some(RunStatus::Gating);
11038 }
11039 let last = reviews.last()?;
11040 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11041 if reviews.len() < max_rounds && !stagnant {
11042 return None;
11043 }
11044 if last.incomplete() && last.blocking == 0 {
11045 return Some(RunStatus::Blocked);
11046 }
11047 if last.e2e_status() == E2eStatus::ResourceBlocked
11048 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11049 {
11050 return None;
11051 }
11052 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11053 RunStatus::Gating
11054 } else {
11055 RunStatus::Blocked
11056 })
11057}
11058
11059pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11074 if nudged {
11075 (full / 4).max(Duration::from_secs(120)).min(full)
11076 } else {
11077 full
11078 }
11079}
11080
11081#[allow(clippy::too_many_arguments)]
11110async fn ask_json_wave<T>(
11111 jobs: Vec<SeatJob>,
11112 sem: Arc<Semaphore>,
11113 retries: usize,
11114 roster: &[AgentSpec],
11115 ctx: &WaveCtx<'_>,
11116 losses: &mut Vec<QuotaLoss>,
11117 state: &mut RunState,
11118 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11119) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11120where
11121 T: serde::de::DeserializeOwned + Send + 'static,
11122{
11123 ask_wave_with(
11124 jobs,
11125 sem,
11126 retries,
11127 roster,
11128 ctx,
11129 losses,
11130 state,
11131 &|text: &str| {
11132 let v = verdict::extract_json::<T>(text)?;
11133 validate(&v)?;
11134 Ok(v)
11135 },
11136 )
11137 .await
11138}
11139
11140#[allow(clippy::too_many_arguments)]
11149async fn ask_wave_with<T>(
11150 jobs: Vec<SeatJob>,
11151 sem: Arc<Semaphore>,
11152 retries: usize,
11153 roster: &[AgentSpec],
11154 ctx: &WaveCtx<'_>,
11155 losses: &mut Vec<QuotaLoss>,
11156 state: &mut RunState,
11157 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11158) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11159where
11160 T: Send + 'static,
11161{
11162 let n = jobs.len();
11163 let originals: Vec<SeatJob> = jobs;
11164 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11165 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11166 let mut nudges: Vec<usize> = vec![0; n];
11173 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11177 let mut tried: Vec<BTreeSet<String>> = specs
11178 .iter()
11179 .map(|s| BTreeSet::from([s.id.clone()]))
11180 .collect();
11181 let starts: Vec<usize> = specs
11182 .iter()
11183 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11184 .collect();
11185 let carry = ctx.carry_seats && !roster.is_empty();
11186 let carried: Vec<BTreeSet<String>> = originals
11189 .iter()
11190 .map(|j| {
11191 state
11192 .seat_history
11193 .get(&j.seat.key)
11194 .filter(|_| carry)
11195 .map(|h| h.failed.clone())
11196 .unwrap_or_default()
11197 })
11198 .collect();
11199 let mut prev: Vec<Option<FailClass>> = originals
11200 .iter()
11201 .map(|j| {
11202 state
11203 .seat_history
11204 .get(&j.seat.key)
11205 .filter(|_| carry)
11206 .and_then(|h| h.last_fail.clone())
11207 })
11208 .collect();
11209 let next_agent =
11210 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11211 let others: BTreeSet<String> = specs
11212 .iter()
11213 .enumerate()
11214 .filter(|(j, _)| *j != i)
11215 .map(|(_, s)| s.id.clone())
11216 .collect();
11217 pick_successor(
11218 roster,
11219 starts[i],
11220 tried,
11221 carry.then(|| &carried[i]),
11222 &others,
11223 )
11224 .cloned()
11225 };
11226 let mut fresh: Vec<Option<String>> = vec![None; n];
11227 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11228 let mut pending: Vec<usize> = (0..n).collect();
11229
11230 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11235 for round in 0..max_rounds {
11236 if pending.is_empty() {
11237 break;
11238 }
11239 let mut batch = Vec::with_capacity(pending.len());
11240 let mut renudged: Vec<&str> = Vec::new();
11241 for &i in &pending {
11242 let src = &originals[i];
11243 let full: &str = match &src.handover {
11248 Some(h) if specs[i].id != src.spec.id => h,
11249 _ => &src.prompt,
11250 };
11251 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11254 (
11255 full.to_owned(),
11256 src.timeout,
11257 format!("{}-{word}-{}", src.stem, specs[i].id),
11258 )
11259 } else if nudges[i] == 0 {
11260 (full.to_owned(), src.timeout, src.stem.clone())
11261 } else {
11262 renudged.push(src.seat.key.as_str());
11263 let why = done[i]
11264 .as_ref()
11265 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11266 .unwrap_or_else(|| "no parsable answer".to_owned());
11267 let nudge = prompt::nudge(&why);
11268 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11269 let prompt = if nudged {
11270 nudge
11271 } else {
11272 format!("{full}\n\n---\n\n{nudge}")
11273 };
11274 (
11275 prompt,
11276 retry_budget(src.timeout, nudged),
11277 format!("{}-retry{}", src.stem, nudges[i]),
11278 )
11279 };
11280 batch.push(SeatJob {
11281 spec: specs[i].clone(),
11282 seat: seats[i].clone(),
11283 cwd: src.cwd.clone(),
11284 prompt,
11285 timeout,
11286 allow_write: src.allow_write,
11287 sessions: src.sessions,
11288 artifacts: src.artifacts.clone(),
11289 stem,
11290 handover: None,
11291 });
11292 }
11293
11294 if !renudged.is_empty() {
11295 state.event(
11296 ctx.node,
11297 format!("retry {round}: re-asking {}", renudged.join(", ")),
11298 );
11299 }
11300 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11301 let mut still = Vec::new();
11302 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11303 seats[i] = seat;
11304 if let AgentOutcome::ResourceBlocked(why) = &out {
11307 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11308 continue;
11309 }
11310 let class = FailClass::of(&out);
11311 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11314 && nudges[i] < retries
11315 && !roster.is_empty();
11316 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11317 let next = should_hand_over(prev[i].as_ref(), &cur)
11318 .then(|| next_agent(i, &tried[i], &specs))
11319 .flatten();
11320 if carry {
11321 let h = state
11322 .seat_history
11323 .entry(originals[i].seat.key.clone())
11324 .or_default();
11325 h.failed.insert(specs[i].id.clone());
11326 h.last_fail = Some(cur.clone());
11327 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11328 h.last_ok = None;
11329 }
11330 if let Err(e) = state.save() {
11333 tracing::warn!("could not persist a seat's failure history: {e:#}");
11334 }
11335 }
11336 if let Some(next) = next {
11337 record_handover(
11338 state,
11339 ctx.node,
11340 &originals[i].seat.key,
11341 &specs[i].id,
11342 &next.id,
11343 &cur,
11344 &fail_reason(&out),
11345 );
11346 tried[i].insert(next.id.clone());
11347 prev[i] = Some(cur.clone());
11348 seats[i] =
11349 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11350 specs[i] = next;
11351 fresh[i] = Some(cur.stem_word().to_owned());
11352 nudges[i] = 0;
11353 done[i] = Some(Err(anyhow::anyhow!(
11354 "handed over after: {}",
11355 fail_reason(&out)
11356 )));
11357 still.push(i);
11358 continue;
11359 }
11360 }
11361 if carry
11362 && !nudge_first
11363 && let Some(cur) = class.clone()
11364 {
11365 let h = state
11367 .seat_history
11368 .entry(originals[i].seat.key.clone())
11369 .or_default();
11370 h.failed.insert(specs[i].id.clone());
11371 h.last_fail = Some(cur);
11372 }
11373 let parsed = match out {
11374 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11375 AgentOutcome::Quota(o) => {
11376 last_quota[i] = o.quota.clone();
11377 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11378 }
11379 AgentOutcome::Dropped(o) => {
11383 let why = o
11384 .dropped
11385 .as_ref()
11386 .map(|d| d.why.as_str())
11387 .unwrap_or("the CLI ended the stream without delivering its answer");
11388 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11389 }
11390 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11391 Err(anyhow::anyhow!(e))
11392 }
11393 };
11394 let quota = class == Some(FailClass::Quota);
11395 let failed = parsed.is_err();
11396 if !failed && matches!(ctx.node, "judge" | "vote") {
11397 state
11400 .quota
11401 .retain(|loss| loss.seat != originals[i].seat.key);
11402 }
11403 done[i] = Some(parsed);
11404 if carry && !failed {
11405 let h = state
11406 .seat_history
11407 .entry(originals[i].seat.key.clone())
11408 .or_default();
11409 h.failed.remove(&specs[i].id);
11410 h.last_ok = Some(specs[i].id.clone());
11411 h.last_fail = None;
11412 }
11413 let agent_failure = class.is_some()
11421 && !nudge_first
11422 && !roster.is_empty()
11423 && next_agent(i, &tried[i], &specs).is_some();
11424 if failed && !quota && !agent_failure && nudges[i] < retries {
11425 nudges[i] += 1;
11426 still.push(i);
11427 }
11428 }
11429 pending = still;
11430 }
11431
11432 for (i, q) in last_quota.into_iter().enumerate() {
11435 if let Some(quota) = q {
11436 losses.push(QuotaLoss::from_report(
11437 originals[i].seat.key.clone(),
11438 ctx.node,
11439 Some("a),
11440 ));
11441 }
11442 }
11443
11444 if state.status != RunStatus::Stalled
11447 && ctx.node != "deliberate"
11448 && n > 0
11449 && losses.len() == n
11450 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11451 {
11452 block_machine_outage(state, ctx.node, losses);
11453 state.quota.append(losses);
11454 }
11455
11456 seats
11457 .into_iter()
11458 .zip(done)
11459 .zip(nudges)
11460 .map(|((seat, res), attempts)| {
11461 (
11462 seat,
11463 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11464 attempts,
11465 )
11466 })
11467 .collect()
11468}
11469
11470pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11473 let mut auth: Vec<_> = losses
11474 .iter()
11475 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11476 .map(|loss| loss.agent.clone())
11477 .collect();
11478 auth.sort();
11479 auth.dedup();
11480 let (kind, detail) = if auth.is_empty() {
11481 (
11482 "agent usage limit",
11483 "All seats reached their agent usage limits; resume after usage is available"
11484 .to_owned(),
11485 )
11486 } else {
11487 (
11488 "agent authentication",
11489 format!(
11490 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11491 auth.join(", ")
11492 ),
11493 )
11494 };
11495 state.resource_block = Some(crate::run::ResourceBlock {
11496 node: node.to_owned(),
11497 kind: kind.to_owned(),
11498 cause: crate::run::ResourceBlockCause::Other,
11499 detail: detail.clone(),
11500 at: Timestamp::now(),
11501 });
11502 state.event(node, detail);
11503}
11504
11505fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11507 results
11508 .iter()
11509 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11510}
11511
11512fn block_wave(
11516 jobs: Vec<SeatJob>,
11517 state: &mut RunState,
11518 node: &str,
11519 attempt: usize,
11520 why: &str,
11521) -> Vec<(usize, SeatState, AgentOutcome)> {
11522 state.resource_wait = None;
11523 state
11524 .active
11525 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11526 if let Err(e) = state.save() {
11527 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11528 }
11529 jobs.into_iter()
11530 .enumerate()
11531 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11532 .collect()
11533}
11534
11535async fn acquire_cache_lease(
11548 state: &mut RunState,
11549 cache_dir: &Path,
11550 owner: &crate::cache::Owner,
11551 budget: Duration,
11552 node: &str,
11553 context: &str,
11554 cancel: &(dyn Fn() -> bool + Sync),
11555) -> Result<Option<crate::cache::Guard>> {
11556 let home = crate::run::home();
11557 let started = Instant::now();
11558 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11559 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11560 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11561 Err(e) => {
11562 state.event(
11563 "verify",
11564 format!("{context}: could not check the shared build cache: {e:#}"),
11565 );
11566 if let Err(e2) = state.save() {
11567 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11568 }
11569 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11570 }
11571 };
11572 state.event(
11573 "verify",
11574 format!(
11575 "{context}: waiting for the shared build cache at {} ({})",
11576 cache_dir.display(),
11577 busy.describe()
11578 ),
11579 );
11580 state.resource_wait = Some(crate::run::ResourceWait {
11583 node: context.to_owned(),
11584 kind: "shared build cache".to_owned(),
11585 holder: busy.describe(),
11586 since: Timestamp::now(),
11587 limit_secs: budget.as_secs(),
11588 });
11589 if let Err(e) = state.save() {
11590 tracing::warn!("could not persist a cache wait: {e:#}");
11591 }
11592 let remaining = budget.saturating_sub(started.elapsed());
11593 let waited = crate::cache::wait_for_unless(
11594 &home,
11595 cache_dir,
11596 owner,
11597 remaining,
11598 Duration::from_secs(5),
11599 cancel,
11600 )
11601 .await;
11602 state.resource_wait = None;
11603 let cause = match &waited {
11604 Ok(Some(_)) => None,
11605 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11606 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11607 Some(crate::run::ResourceBlockCause::Timeout {
11608 waited_secs: started.elapsed().as_secs(),
11609 limit_secs: budget.as_secs(),
11610 })
11611 }
11612 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11613 };
11614 if let Some(cause) = cause {
11615 let detail = match &waited {
11616 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11617 Err(error) => format!("{error:#}"),
11618 };
11619 state.resource_block = Some(crate::run::ResourceBlock {
11620 cause,
11621 node: node.to_owned(),
11622 kind: "shared build cache".into(),
11623 detail,
11624 at: Timestamp::now(),
11625 });
11626 state.event(
11627 "resource",
11628 state.resource_block.as_ref().unwrap().description(),
11629 );
11630 }
11631 if let Err(error) = state.save() {
11632 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11633 }
11634 waited
11635}
11636
11637#[derive(Debug)]
11642struct CacheCheckError(String);
11643
11644impl std::fmt::Display for CacheCheckError {
11645 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11646 f.write_str(&self.0)
11647 }
11648}
11649
11650impl std::error::Error for CacheCheckError {}
11651
11652fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11654 state
11655 .resource_block
11656 .get_or_insert_with(|| crate::run::ResourceBlock {
11657 cause: crate::run::ResourceBlockCause::Other,
11658 node: node.to_owned(),
11659 kind: "shared build cache".into(),
11660 detail: format!("{error:#}"),
11661 at: Timestamp::now(),
11662 });
11663 if let Err(error) = state.save() {
11664 tracing::warn!("could not persist cache resource block: {error:#}");
11665 }
11666}
11667
11668#[allow(clippy::too_many_arguments)]
11689async fn with_cache_lease<'s, F, Fut>(
11690 state: &'s mut RunState,
11691 cache_dir: Option<&Path>,
11692 node: &str,
11693 seat: &str,
11694 worktree: &Path,
11695 head: &str,
11696 budget: Duration,
11697 context: &str,
11698 wait: &crate::repo_lock::WaitPolicy,
11699 body: F,
11700) -> Result<(Vec<CommandOutcome>, bool)>
11701where
11702 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11703 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11704{
11705 let started = Instant::now();
11708 let home = crate::run::home();
11709 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11710 let guard = if let Some(cache_dir) = cache_dir {
11711 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11712 match acquire_cache_lease(
11713 state,
11714 cache_dir,
11715 &owner,
11716 resource_limit,
11717 node,
11718 context,
11719 &|| wait.cancelled(),
11720 )
11721 .await
11722 .and_then(|g| {
11723 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11724 }) {
11725 Ok(g) => Some(g),
11726 Err(e) => {
11727 record_cache_error(state, node, &e);
11728 return Ok((
11729 vec![CommandOutcome {
11730 command: "(waiting for the shared build cache)".into(),
11731 code: None,
11732 output_tail: e.to_string(),
11733 duration_ms: started.elapsed().as_millis() as u64,
11734 resource_blocked: true,
11735 }],
11736 false,
11737 ));
11738 }
11739 }
11740 } else {
11741 None
11742 };
11743 let _coordination = coordinate_operation(state, worktree, wait).await?;
11744 if let Some(cache_dir) = cache_dir {
11745 let identity = crate::cache::Identity::new(worktree, head);
11746 if let Err(e) = crate::cache::ensure_fresh(
11747 &home,
11748 cache_dir,
11749 &identity,
11750 &state.config.verify.cargo_manifests(),
11751 ) {
11752 record_cache_error(state, node, &e);
11753 state.event(
11754 "verify",
11755 format!(
11756 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11757 worktree.display(),
11758 short(head)
11759 ),
11760 );
11761 if let Some(guard) = guard {
11762 guard.release();
11763 }
11764 return Ok((
11765 vec![CommandOutcome {
11766 command: "(confirming the shared build cache is fresh)".into(),
11767 code: None,
11768 output_tail: e.to_string(),
11769 duration_ms: started.elapsed().as_millis() as u64,
11770 resource_blocked: true,
11771 }],
11772 false,
11773 ));
11774 }
11775 }
11776 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11777 if !timed_out_pids.is_empty() {
11778 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11779 }
11780 drop(_coordination);
11783 if let Some(guard) = guard {
11784 guard.release();
11785 }
11786 Ok((outcomes, retried))
11787}
11788
11789async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11801 wait_for_pids_with(
11802 pids,
11803 crate::proc::pid_alive,
11804 LEASE_RELEASE_POLL,
11805 LEASE_RELEASE_MAX_WAIT,
11806 )
11807 .await;
11808}
11809
11810async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11816 pids: &[u32],
11817 alive: F,
11818 poll: Duration,
11819 max_wait: Duration,
11820) {
11821 let deadline = Instant::now() + max_wait;
11822 loop {
11823 if pids.iter().all(|&pid| !alive(pid)) {
11824 return;
11825 }
11826 if Instant::now() >= deadline {
11827 return;
11828 }
11829 tokio::time::sleep(poll).await;
11830 }
11831}
11832
11833fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11840 outcomes.iter().any(|o| o.resource_blocked)
11841}
11842
11843pub(crate) enum GateFix {
11845 Retry,
11847 Stop,
11850 Defer,
11853}
11854
11855pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11863 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11864 red.peek().is_some()
11865 && red.all(|o| {
11866 !o.resource_blocked
11867 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11868 && !o.output_tail.trim().is_empty()
11869 })
11870}
11871
11872fn e2e_outcome_label(o: &CommandOutcome) -> String {
11876 if o.ok() {
11877 return "pass".to_owned();
11878 }
11879 let reason = if o.build_failed() {
11880 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11881 } else {
11882 format!("FAIL ({:?})", o.code)
11883 };
11884 format!(
11885 "{reason}\n{}",
11886 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11887 )
11888}
11889
11890async fn run_e2e_with_retry(
11898 state: &mut RunState,
11899 shell: &[String],
11900 commands: &[String],
11901 worktree: &Path,
11902 timeout: Duration,
11903 context: &str,
11904) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11905 let (mut e2e, mut timed_out_pids) = run_commands(
11906 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11907 )
11908 .await;
11909 for o in &e2e {
11910 state.event(
11911 "verify",
11912 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11913 );
11914 }
11915 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11919 if verify_retried {
11920 state.event(
11921 "verify",
11922 format!(
11923 "{context}: verify could not build/link, not a test result — retrying once \
11924 before concluding"
11925 ),
11926 );
11927 let retried = run_commands(
11928 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11929 )
11930 .await;
11931 e2e = retried.0;
11932 timed_out_pids.extend(retried.1);
11935 for o in &e2e {
11936 state.event(
11937 "verify",
11938 format!(
11939 "{context}: retry `{}` -> {}",
11940 o.command,
11941 e2e_outcome_label(o)
11942 ),
11943 );
11944 }
11945 }
11946 (e2e, verify_retried, timed_out_pids)
11947}
11948
11949#[allow(clippy::too_many_arguments)]
11965async fn run_commands(
11966 state: &mut RunState,
11967 node: &str,
11968 task: &str,
11969 attempt: usize,
11970 shell: &[String],
11971 commands: &[String],
11972 cwd: &Path,
11973 timeout: Duration,
11974) -> (Vec<CommandOutcome>, Vec<u32>) {
11975 if commands.is_empty() {
11976 return (Vec::new(), Vec::new());
11981 }
11982 let mut out = Vec::new();
11983 let mut timed_out_pids = Vec::new();
11984 let total = commands.len();
11985 for (idx, command) in commands.iter().enumerate() {
11986 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11987 if let Err(e) = state.save() {
11988 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11989 }
11990 let started = Instant::now();
11991 let mut cmd = tokio::process::Command::new(&shell[0]);
11992 cmd.quiet();
11993 for key in crate::worker::INVOCATION_ENV {
11997 cmd.env_remove(key);
11998 }
11999 cmd.args(&shell[1..])
12000 .arg(command)
12001 .current_dir(cwd)
12002 .stdin(std::process::Stdio::null())
12003 .stdout(std::process::Stdio::piped())
12004 .stderr(std::process::Stdio::piped())
12005 .kill_on_drop(true);
12006 let mut outputs = Vec::new();
12007 let protected = (|| -> Result<()> {
12008 crate::incremental::configure(&mut cmd, cwd)?;
12009 let mut paths = vec![cwd.to_owned()];
12010 if let Some(cache) = state.config.cache_dir() {
12011 paths.push(cache);
12012 }
12013 outputs.extend(crate::retention::OutputLease::admit(
12014 &crate::run::home(),
12015 &paths,
12016 state.config.disk.min_free_bytes,
12017 )?);
12018 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12019 outputs.push(guard);
12020 }
12021 if let Some(cache) = state.config.cache_dir() {
12022 outputs.push(crate::retention::OutputLease::cache(
12023 &crate::run::home(),
12024 &cache,
12025 false,
12026 )?);
12027 }
12028 Ok(())
12029 })();
12030 if let Err(e) = protected {
12031 let detail = format!("execution admission unavailable: {e:#}");
12032 state.resource_block = Some(crate::run::ResourceBlock {
12033 cause: crate::run::ResourceBlockCause::Other,
12034 node: node.to_owned(),
12035 kind: if detail.contains("disk admission") {
12036 "disk admission"
12037 } else {
12038 "output ownership"
12039 }
12040 .to_owned(),
12041 detail: detail.clone(),
12042 at: Timestamp::now(),
12043 });
12044 out.push(CommandOutcome {
12045 command: command.clone(),
12046 code: None,
12047 output_tail: detail,
12048 duration_ms: 0,
12049 resource_blocked: true,
12050 });
12051 break;
12052 }
12053 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12054 let (code, body) = match spawned {
12055 Ok(mut running) => {
12056 let pid = running.child().id();
12057 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12058 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12059 let mut paths = vec![cwd.to_owned()];
12060 if let Some(cache) = state.config.cache_dir() {
12061 paths.push(cache);
12062 }
12063 let mut pressure = None;
12064 let status = tokio::select! {
12065 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12066 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12067 pressure = Some(reason); None
12068 }
12069 };
12070 let (code, failure) = match status {
12071 Some(Ok(Ok(status))) => (
12072 status.code(),
12073 status
12074 .code()
12075 .is_none()
12076 .then(|| format!("command terminated: {status}")),
12077 ),
12078 Some(Ok(Err(error))) => {
12079 (None, Some(format!("failed waiting for command: {error}")))
12080 }
12081 Some(Err(_)) => {
12082 if !running.coordinated()
12086 && let Some(pid) = pid
12087 {
12088 timed_out_pids.push(pid);
12089 }
12090 (
12091 None,
12092 Some(format!("timed out after {}s", timeout.as_secs())),
12093 )
12094 }
12095 None => (None, None),
12096 };
12097 running.finish().await;
12098 if let Some(reason) = pressure {
12099 state.resource_block = Some(crate::run::ResourceBlock {
12100 cause: crate::run::ResourceBlockCause::Other,
12101 node: node.to_owned(),
12102 kind: "disk pressure".to_owned(),
12103 detail: reason.clone(),
12104 at: Timestamp::now(),
12105 });
12106 out.push(CommandOutcome {
12107 command: command.clone(),
12108 code: None,
12109 output_tail: reason,
12110 duration_ms: started.elapsed().as_millis() as u64,
12111 resource_blocked: true,
12112 });
12113 break;
12114 }
12115 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12116 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12117 if let Some(failure) = failure {
12118 body.push('\n');
12119 body.push_str(&failure);
12120 }
12121 (code, body)
12122 }
12123 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12124 };
12125 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12126 if !body.is_empty() {
12127 match crate::run::write_verification_artifact(state, task, &body) {
12131 Ok(name) if summary != body => {
12132 summary = format!("[full output: artifacts/{name}]\n{summary}");
12133 }
12134 Ok(_) => {}
12135 Err(e) => {
12136 let warning = format!("could not save the full {task} output: {e:#}");
12137 tracing::warn!("{warning}");
12138 state.event(node, warning);
12139 if summary != body {
12140 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12141 }
12142 }
12143 }
12144 }
12145 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12146 out.push(CommandOutcome {
12147 command: command.clone(),
12148 code,
12149 output_tail: summary,
12150 duration_ms: started.elapsed().as_millis() as u64,
12151 resource_blocked,
12152 });
12153 if resource_blocked {
12154 state.resource_block = Some(crate::run::ResourceBlock {
12155 cause: crate::run::ResourceBlockCause::Other,
12156 node: node.to_owned(),
12157 kind: "incremental cache admission".into(),
12158 detail: body,
12159 at: Timestamp::now(),
12160 });
12161 break;
12162 }
12163 }
12164 state.task_finished(task);
12165 if let Err(e) = state.save() {
12166 tracing::warn!("could not persist the end of {task}: {e:#}");
12167 }
12168 (out, timed_out_pids)
12169}
12170
12171fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12183 let repo = repo.display();
12184 match style {
12185 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12186 MergeStyle::Squash => {
12187 let subject = message
12190 .lines()
12191 .next()
12192 .unwrap_or(branch)
12193 .replace(['\\', '"', '$', '`'], "");
12194 format!(
12195 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12196 )
12197 }
12198 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12199 }
12200}
12201
12202const PR_TITLE_MAX: usize = 240;
12213
12214fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12225 let line = queue::first_line(text)?;
12226 let chars: Vec<char> = line.chars().collect();
12227 let (kept, mark) = if chars.len() <= max {
12228 (line.to_owned(), "")
12229 } else if let Some(end) = sentence_end(&chars, max) {
12230 (chars[..end].iter().collect::<String>(), "")
12231 } else {
12232 let room = max.saturating_sub(3);
12233 let cut = if chars[room].is_whitespace() {
12236 room
12237 } else {
12238 chars[..room]
12239 .iter()
12240 .rposition(|c| c.is_whitespace())
12241 .unwrap_or(room)
12242 };
12243 let head: String = chars[..cut].iter().collect();
12244 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12245 (head.to_owned(), "...")
12246 };
12247 if kept.is_empty() {
12248 return None;
12249 }
12250 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12251 return None;
12252 }
12253 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12254 return None;
12255 }
12256 Some(format!("{kept}{mark}"))
12257}
12258
12259fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12263 const MIN: usize = 20;
12264 for i in MIN..max.min(chars.len()) {
12265 if !matches!(chars[i], '.' | '!' | '?') {
12266 continue;
12267 }
12268 let Some(&next) = chars.get(i + 1) else {
12269 continue;
12270 };
12271 if !next.is_whitespace() {
12272 continue;
12273 }
12274 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12275 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12276 continue;
12277 }
12278 let word: String = chars[..i]
12279 .iter()
12280 .rev()
12281 .take_while(|c| !c.is_whitespace())
12282 .collect::<Vec<_>>()
12283 .into_iter()
12284 .rev()
12285 .collect();
12286 let word = word.to_ascii_lowercase();
12287 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12288 continue;
12289 }
12290 let end = chars[..i]
12291 .iter()
12292 .rposition(|c| !c.is_whitespace())
12293 .map_or(i, |p| p + 1);
12294 return Some(end);
12295 }
12296 None
12297}
12298
12299struct PrMessage {
12304 title: String,
12305 body: String,
12306}
12307
12308impl PrMessage {
12309 fn commit_message(&self) -> String {
12313 format!("{}\n\n{}", self.title, self.body)
12314 }
12315}
12316
12317fn title_marker(line: &str) -> Option<&str> {
12319 let line = line.trim();
12320 let head = line.get(..6)?;
12321 head.eq_ignore_ascii_case("title:")
12322 .then(|| line[6..].trim())
12323}
12324
12325fn summary_title(summary: &str) -> Option<String> {
12330 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12331 let raw = title_marker(first)?;
12332 if raw.is_empty() {
12333 return None;
12334 }
12335 let title = queue::title_from(raw, PR_TITLE_MAX);
12336 let lower = title.to_ascii_lowercase();
12337 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12338 return None;
12339 }
12340 Some(title)
12341}
12342
12343const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12345
12346pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12348
12349pub(crate) fn is_review_run(state: &RunState) -> bool {
12353 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12354}
12355
12356fn review_title(state: &RunState) -> Option<String> {
12361 english_subject(
12362 state.reviewed_commits.as_ref()?.first()?,
12363 state.config.graph.github_english(),
12364 )
12365}
12366
12367fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12370 let raw = raw.trim();
12371 if raw.is_empty() {
12372 return None;
12373 }
12374 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12375 return None;
12376 }
12377 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12378 return None;
12379 }
12380 let title = queue::title_from(raw, PR_TITLE_MAX);
12381 let lower = title.to_ascii_lowercase();
12382 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12383 return None;
12384 }
12385 Some(title)
12386}
12387
12388#[derive(Debug, Clone, PartialEq, Eq)]
12391struct BranchFacts {
12392 commits: Vec<(String, String)>,
12394 stat: String,
12396}
12397
12398const STAT_FILE_LINES: usize = 25;
12400const COMMITS_MAX_CHARS: usize = 20_000;
12402
12403async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12406 let commits = git::commit_log(repo, base, branch).await.ok()?;
12407 if commits.is_empty() {
12408 return None;
12409 }
12410 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12411 let lines: Vec<&str> = stat.lines().collect();
12412 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12413 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12414 let more = format!(" ... {omitted} more file(s)");
12415 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12416 kept.push(&more);
12417 kept.push(lines[lines.len() - 1]);
12418 kept.join("\n")
12419 } else {
12420 lines.join("\n")
12421 };
12422 Some(BranchFacts { commits, stat })
12423}
12424
12425fn markdown_safe(text: &str) -> String {
12428 text.replace("</details>", "</details>")
12429 .replace("\x60\x60\x60", "~~~")
12430}
12431
12432fn neutral_title(state: &RunState, winner: char) -> String {
12433 format!(
12434 "chore: land candidate {} of run {}",
12435 winner.to_ascii_uppercase(),
12436 state.id
12437 )
12438}
12439
12440pub(crate) async fn repair_landing_gate(
12443 state: &mut RunState,
12444 winner: &Candidate,
12445 outcomes: &[CommandOutcome],
12446) -> Result<GateFix> {
12447 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12448 return Ok(GateFix::Stop);
12449 }
12450 let mut runner = Runner {
12451 roles: state.config.resolve_roles()?,
12452 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12453 state: state.clone(),
12454 pause: Pause::new(),
12455 interrupt: Pause::new(),
12456 };
12457 let result = runner.gate_fix_round(winner, outcomes).await;
12458 *state = runner.state;
12459 result
12460}
12461
12462pub(crate) async fn verify_current_head(
12465 state: &mut RunState,
12466 winner: &Candidate,
12467 wait: &crate::repo_lock::WaitPolicy,
12468) -> Result<bool> {
12469 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12470 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12472 state
12473 .verification_history
12474 .push(crate::run::VerificationEvidence {
12475 superseded_by: Some(head.clone()),
12476 archived_at: Some(Timestamp::now()),
12477 fixes_spent: Some(state.gate_fixes.len()),
12478 gate_head: None,
12479 gate_ran: false,
12480 gate: Vec::new(),
12481 e2e: state.head_e2e.take(),
12482 });
12483 }
12484 if let Some(e) = &state.head_e2e
12485 && e.head == head
12486 {
12487 if e.passed {
12488 return Ok(true);
12489 }
12490 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12491 state.status = RunStatus::Blocked;
12492 state.save()?;
12493 return Ok(false);
12494 }
12495 } else if state.reviews.last().is_some_and(|r| {
12496 r.verified_head.as_deref() == Some(head.as_str())
12497 && (r.e2e_status() == E2eStatus::Passed
12498 || (r.e2e_status() == E2eStatus::NotConfigured
12499 && state.config.verify.e2e.is_empty()))
12500 }) || state
12501 .gate_repair_e2e
12502 .as_ref()
12503 .is_some_and(|e| e.head == head && e.passed)
12504 {
12505 return Ok(true);
12506 }
12507 if let Some(e) = &state.gate_repair_e2e
12508 && e.head == head
12509 && !e.passed
12510 && !e.outcomes.is_empty()
12511 && !verify_inconclusive(&e.outcomes)
12512 {
12513 state.status = RunStatus::Blocked;
12514 state.save()?;
12515 return Ok(false);
12516 }
12517 state.head_e2e = Some(crate::run::GateRepairE2e {
12518 head: head.clone(),
12519 passed: false,
12520 outcomes: Vec::new(),
12521 });
12522 state.save()?;
12523 let commands = state.config.verify.e2e.clone();
12524 let shell = state.config.shell();
12525 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12526 let cache_dir = state.config.cache_dir();
12527 let outcomes = if commands.is_empty() {
12528 Vec::new()
12529 } else {
12530 let (outcomes, _) = with_cache_lease(
12531 state,
12532 cache_dir.as_deref(),
12533 "e2e",
12534 "e2e",
12535 &winner.worktree,
12536 &head,
12537 timeout,
12538 "current head e2e",
12539 wait,
12540 |state, budget| async move {
12541 run_e2e_with_retry(
12542 state,
12543 &shell,
12544 &commands,
12545 &winner.worktree,
12546 budget,
12547 "current head e2e",
12548 )
12549 .await
12550 },
12551 )
12552 .await?;
12553 outcomes
12554 };
12555 let unresolved = verify_inconclusive(&outcomes);
12556 let passed = !unresolved
12557 && outcomes.iter().all(CommandOutcome::ok)
12558 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12559 if !outcomes.is_empty() && !unresolved {
12560 state.record_base_progress(&head);
12561 }
12562 state.head_e2e = Some(crate::run::GateRepairE2e {
12563 head: head.clone(),
12564 passed,
12565 outcomes,
12566 });
12567 if !passed && !unresolved {
12568 state.status = RunStatus::Blocked;
12569 state.event(
12570 "e2e",
12571 format!("verify.e2e failed on head {head}; nothing published"),
12572 );
12573 }
12574 state.save()?;
12575 Ok(passed)
12576}
12577
12578pub(crate) async fn check_gate_repair_e2e(
12581 state: &mut RunState,
12582 winner: &Candidate,
12583 wait: &crate::repo_lock::WaitPolicy,
12584) -> Result<GateFix> {
12585 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12586 return Ok(GateFix::Retry);
12587 }
12588 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12589 if state
12590 .gate_repair_e2e
12591 .as_ref()
12592 .is_some_and(|r| r.head == head && r.passed)
12593 {
12594 return Ok(GateFix::Retry);
12595 }
12596 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12597 head: head.clone(),
12598 passed: false,
12599 outcomes: Vec::new(),
12600 });
12601 state.save()?;
12602 let commands = state.config.verify.e2e.clone();
12603 let outcomes = if commands.is_empty() {
12604 Vec::new()
12605 } else {
12606 let shell = state.config.shell();
12607 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12608 let cache_dir = state.config.cache_dir();
12609 let (outcomes, _) = with_cache_lease(
12610 state,
12611 cache_dir.as_deref(),
12612 "e2e",
12613 "e2e",
12614 &winner.worktree,
12615 &head,
12616 timeout,
12617 "gate repair e2e",
12618 wait,
12619 |state, budget| async move {
12620 run_e2e_with_retry(
12621 state,
12622 &shell,
12623 &commands,
12624 &winner.worktree,
12625 budget,
12626 "gate repair e2e",
12627 )
12628 .await
12629 },
12630 )
12631 .await?;
12632 outcomes
12633 };
12634 let deferred = verify_inconclusive(&outcomes);
12635 let passed = outcomes.iter().all(CommandOutcome::ok)
12636 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12637 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12638 head,
12639 passed,
12640 outcomes,
12641 });
12642 state.save()?;
12643 if deferred {
12644 Ok(GateFix::Defer)
12645 } else if !passed {
12646 state.event(
12647 "gate",
12648 "verify.e2e failed after gate repair; nothing published",
12649 );
12650 Ok(GateFix::Stop)
12651 } else {
12652 Ok(GateFix::Retry)
12653 }
12654}
12655
12656pub(crate) async fn run_landing_gate(
12658 state: &mut RunState,
12659 winner: &Candidate,
12660 wait: &crate::repo_lock::WaitPolicy,
12661) -> Result<Vec<CommandOutcome>> {
12662 let shell = state.config.shell();
12663 let gate_commands = state.config.verify.gate.clone();
12664 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12665 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12666 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12667 Vec::new()
12668 } else {
12669 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12670 let cache_dir = state.config.cache_dir();
12671 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12672 let (outcomes, _) = with_cache_lease(
12673 state,
12674 cache_dir.as_deref(),
12675 "gate",
12676 "gate",
12677 &winner.worktree,
12678 &head,
12679 timeout,
12680 "final gate",
12681 wait,
12682 |state, budget| {
12683 let shell = shell.clone();
12684 let gate_commands = gate_commands.clone();
12685 let winner = winner.clone();
12686 async move {
12687 Runner::run_pre_gate(state, &winner).await;
12688 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12690 let (outcomes, timed_out_pids) = run_commands(
12691 state,
12692 "gate",
12693 "gate",
12694 0,
12695 &shell,
12696 &gate_commands,
12697 &winner.worktree,
12698 budget,
12699 )
12700 .await;
12701 (outcomes, false, timed_out_pids)
12702 }
12703 },
12704 )
12705 .await?;
12706 outcomes
12707 };
12708 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12709 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12710 let mut outcomes = outcomes;
12711 outcomes.push(CommandOutcome {
12712 command: "gate head stability".into(), code: None,
12713 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12714 duration_ms: 0, resource_blocked: false,
12715 });
12716 return Ok(outcomes);
12717 }
12718 if outcomes.is_empty() {
12719 state.event("gate", "no gate commands configured; not tested");
12724 }
12725 for o in &outcomes {
12726 state.event(
12727 "gate",
12728 format!(
12729 "`{}` -> {}",
12730 o.command,
12731 if o.ok() {
12732 "pass".to_owned()
12733 } else {
12734 format!(
12735 "FAIL ({:?})\n{}",
12736 o.code,
12737 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12738 )
12739 }
12740 ),
12741 );
12742 }
12743 Ok(outcomes)
12744}
12745
12746pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12752 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12753 ""
12754 } else {
12755 pr_title
12756 }
12757}
12758
12759pub fn landing_subject_source(state: &RunState) -> String {
12763 if is_review_run(state) {
12764 let winner = state.candidates.first().map_or('A', |c| c.label);
12765 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12766 }
12767 state.instruction.clone()
12768}
12769
12770fn summary_without_title(summary: &str) -> String {
12773 let mut lines = summary.trim().lines().peekable();
12774 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12775 lines.next();
12776 }
12777 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12778}
12779
12780#[cfg(test)]
12792fn pr_message(state: &RunState, winner: char) -> PrMessage {
12793 pr_message_with(state, winner, None)
12794}
12795
12796#[cfg(test)]
12799fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12800 let raw = pr_message_raw(
12801 state,
12802 winner,
12803 facts,
12804 crate::pr_outcome::FinalHead::Unreported,
12805 ChangeNote::NotAsked,
12806 );
12807 let id = crate::scrub::Identity::current();
12808 PrMessage {
12809 title: crate::scrub::scrub(&raw.title, &id),
12810 body: crate::scrub::scrub(&raw.body, &id),
12811 }
12812}
12813
12814#[derive(Debug, Clone, Copy)]
12816enum ChangeNote<'a> {
12817 NotAsked,
12819 Summary(&'a crate::run::ChangeSummary),
12821 Unavailable(&'a str),
12823}
12824
12825fn pr_message_raw(
12826 state: &RunState,
12827 winner: char,
12828 facts: Option<&BranchFacts>,
12829 head: crate::pr_outcome::FinalHead<'_>,
12830 change: ChangeNote<'_>,
12831) -> PrMessage {
12832 if !matches!(change, ChangeNote::NotAsked) {
12833 let (title, mut body) = match change {
12834 ChangeNote::Summary(c) => (
12835 c.title.clone(),
12836 format!("## Summary\n\n{}\n\n", c.body.trim()),
12837 ),
12838 ChangeNote::Unavailable(why) => (
12839 neutral_title(state, winner),
12840 format!(
12841 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12842 ),
12843 ),
12844 ChangeNote::NotAsked => unreachable!(),
12845 };
12846 body.push_str(&public_pr_evidence(state, head));
12847 body.push_str(&format!(
12848 "\n---\nmagi:run/{} magi:candidate-{}\n",
12849 state.id,
12850 winner.to_ascii_lowercase()
12851 ));
12852 return PrMessage { title, body };
12853 }
12854 pr_preview(state, winner, facts, head)
12855}
12856
12857fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12860 let mut body = crate::pr_outcome::review_rounds(state);
12861 body.push_str(&crate::pr_outcome::final_verification(state, head));
12862 let open = state.last_round_findings();
12863 if !open.is_empty() {
12864 body.push_str("## Open review findings\n\n");
12865 for f in open {
12866 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12867 }
12868 body.push('\n');
12869 }
12870 if let Some(c) = state.change_summary.as_ref()
12871 && head == crate::pr_outcome::FinalHead::At(&c.head)
12872 {
12873 body.push_str(&format!(
12874 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12875 c.base, c.head
12876 ));
12877 }
12878 body
12879}
12880
12881fn pr_preview(
12883 state: &RunState,
12884 winner: char,
12885 facts: Option<&BranchFacts>,
12886 head: crate::pr_outcome::FinalHead<'_>,
12887) -> PrMessage {
12888 let summary = state
12889 .candidates
12890 .iter()
12891 .find(|c| c.label == winner)
12892 .map(|c| c.summary.as_str())
12893 .unwrap_or_default();
12894 let review = is_review_run(state);
12897 let title = if review {
12898 facts
12899 .and_then(|f| {
12900 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12901 })
12902 .or_else(|| review_title(state))
12903 .or_else(|| {
12904 state
12905 .candidates
12906 .iter()
12907 .find(|c| c.label == winner)
12908 .filter(|c| !c.branch.starts_with("magi/"))
12909 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12910 })
12911 .unwrap_or_else(|| neutral_title(state, winner))
12912 } else {
12913 summary_title(summary).unwrap_or_else(|| {
12914 english_title(
12915 &state.instruction,
12916 PR_TITLE_MAX,
12917 state.config.graph.github_english(),
12918 )
12919 .unwrap_or_else(|| neutral_title(state, winner))
12920 })
12921 };
12922
12923 let mut body = String::new();
12924 let what = summary_without_title(summary);
12925 if !what.is_empty() {
12926 body.push_str("## Summary\n\n");
12927 body.push_str(&what);
12928 body.push_str("\n\n");
12929 }
12930
12931 body.push_str(&crate::pr_outcome::review_rounds(state));
12933 body.push_str(&crate::pr_outcome::final_verification(state, head));
12934 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12935
12936 let open = state.open_findings();
12937 if !open.is_empty() {
12938 body.push_str("## Open review findings\n\n");
12939 for f in &open {
12940 body.push_str(&format!(
12941 "- `{}` [{:?}] {}\n",
12942 f.id, f.severity, "Context retained in local review records."
12943 ));
12944 }
12945 body.push('\n');
12946 }
12947
12948 if let Some(fix) = fix
12949 && !fix.rejected.is_empty()
12950 {
12951 body.push_str("## Declined by the fixer\n\n");
12952 for r in &fix.rejected {
12953 body.push_str(&format!(
12954 "- `{}`: {}\n",
12955 r.id, "Reason retained in local review records."
12956 ));
12957 }
12958 body.push('\n');
12959 }
12960
12961 if review {
12962 body.push_str("## Commits under review\n\n");
12964 if let Some(facts) = facts {
12965 let mut left = COMMITS_MAX_CHARS;
12966 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12967 let mut entry = format!("- {}\n", markdown_safe(subject));
12968 for l in markdown_safe(text).lines() {
12969 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12970 }
12971 if left == 0 {
12972 body.push_str(&format!(
12973 "- ... {} more commit(s)\n",
12974 facts.commits.len() - i
12975 ));
12976 break;
12977 }
12978 if entry.len() > left {
12979 let mut end = left;
12982 while !entry.is_char_boundary(end) {
12983 end -= 1;
12984 }
12985 entry.truncate(end);
12986 entry.push_str("\n ... (truncated)\n");
12987 left = 0;
12988 } else {
12989 left -= entry.len();
12990 }
12991 body.push_str(&entry);
12992 }
12993 if !facts.stat.trim().is_empty() {
12994 body.push_str(&format!(
12995 "\n## Diff stat\n\n```\n{}\n```\n",
12996 markdown_safe(facts.stat.trim())
12997 ));
12998 }
12999 } else {
13000 match &state.reviewed_commits {
13001 Some(subjects) => {
13002 for s in subjects {
13003 body.push_str(&format!("- {}\n", s.trim()));
13004 }
13005 }
13006 None => {
13007 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13010 body.push_str(rest.trim());
13011 body.push('\n');
13012 }
13013 }
13014 }
13015 } else {
13016 let task = state.instruction.trim();
13017 let task = if task.is_empty() {
13018 "(empty task)"
13019 } else {
13020 task
13021 };
13022 body.push_str(&format!(
13023 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13024 task.replace("</details>", "</details>")
13025 ));
13026 }
13027
13028 body.push_str(&format!(
13029 "\n---\nmagi:run/{} magi:candidate-{}\n",
13030 state.id,
13031 winner.to_ascii_lowercase()
13032 ));
13033
13034 PrMessage { title, body }
13035}
13036
13037fn seeded_instruction(state: &RunState) -> String {
13040 let mut inputs = Vec::new();
13041 if let Some(start) = &state.sources.start_from {
13042 inputs.push(format!(
13043 "Start from `{}` at `{}` (including ancestry).",
13044 start.token, start.sha
13045 ));
13046 }
13047 for pick in &state.sources.pick_commits {
13048 inputs.push(format!(
13049 "Pick `{}` at `{}` (single commit).",
13050 pick.token, pick.sha
13051 ));
13052 }
13053 if inputs.is_empty() {
13054 state.instruction.clone()
13055 } else {
13056 format!(
13057 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13058 state.instruction,
13059 inputs.join("\n")
13060 )
13061 }
13062}
13063
13064async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13068 let base = &state.base_branch;
13069 let mut against = base.clone();
13070 if mode == MergeMode::Pr {
13071 let remote = &state.config.merge.remote;
13075 let tracking = format!("{remote}/{base}");
13076 let fetched = git::fetch(repo, remote, base).await;
13077 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13078 against = tracking;
13079 } else {
13080 return false;
13081 }
13082 }
13083 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13084}
13085
13086fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13089 let mut detail = format!(
13090 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13091 nothing to open a pull request for"
13092 );
13093 detail.push_str(
13094 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13095 );
13096 detail
13097}
13098
13099#[derive(Debug, PartialEq, Eq)]
13102enum PrPlan {
13103 Create,
13104 Adopt { url: String, title: String },
13105 Stop(String),
13106}
13107
13108fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13111 match found {
13112 Ok(land::OpenPr::None) => PrPlan::Create,
13113 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13114 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13115 "several open pull requests exist for this branch, not picking one: {}",
13116 urls.join(" ")
13117 )),
13118 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13119 }
13120}
13121
13122pub(crate) async fn gh_pr_create(
13124 cwd: &Path,
13125 base: &str,
13126 head: &str,
13127 title: &str,
13128 body: &str,
13129) -> Result<String> {
13130 let out = tokio::process::Command::new("gh")
13131 .args([
13132 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13133 ])
13134 .current_dir(cwd)
13135 .quiet()
13136 .stdin(std::process::Stdio::null())
13137 .output()
13138 .await
13139 .context("spawn gh")?;
13140 if out.status.success() {
13141 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13142 } else {
13143 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13144 }
13145}
13146
13147async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13149 match std::fs::symlink_metadata(path) {
13150 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13151 git::worktree_remove_clean(repo, path).await?;
13154 return Ok(());
13155 }
13156 Err(e) => return Err(e.into()),
13157 Ok(_) => {}
13158 }
13159 if !git::is_clean(path).await? {
13160 bail!(
13161 "worktree has uncommitted work; refusing removal: {}",
13162 path.display()
13163 );
13164 }
13165 if !git::worktree_remove_clean(repo, path).await? {
13166 bail!("worktree removal refused: {}", path.display());
13167 }
13168 Ok(())
13169}
13170
13171pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13180 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13181 bail!("refusing fold: {reason}");
13182 }
13183 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13184 bail!("candidate preservation incomplete; refusing cleanup");
13185 }
13186 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13190 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13191 let _driver = crate::driver::maintenance(home, &state.id)?;
13192 let mut output_guards = Vec::new();
13193 for entry in std::fs::read_dir(state.worktree_root())
13194 .into_iter()
13195 .flatten()
13196 .flatten()
13197 {
13198 if entry.path().join(".git").is_file() {
13199 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13200 }
13201 }
13202 let repo = state.repo.clone();
13203 let wait = repository_wait(state, None);
13206 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13207 Ok(lease) => lease,
13208 Err(e) => {
13209 state.save_under(home)?;
13210 return Err(e);
13211 }
13212 };
13213 let _source_outputs = if state.status == RunStatus::Merged
13216 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13217 {
13218 verify_merged_fold_sources(state, drop_winner, home).await?
13219 } else {
13220 Vec::new()
13221 };
13222 let root = state.worktree_root();
13223 let winner = state.tally.as_ref().map(|t| t.winner);
13224 let mut removed = Vec::new();
13225
13226 for i in 0..state.candidates.len() {
13227 let c = state.candidates[i].clone();
13228 let is_winner = Some(c.label) == winner;
13229 if is_winner && !drop_winner {
13230 continue;
13231 }
13232 let existed = c.worktree.exists();
13233 remove_preserved_worktree(&repo, &c.worktree).await?;
13236 if existed {
13237 removed.push(c.worktree.to_string_lossy().into_owned());
13238 }
13239 let handed_over = state.released_branches.contains(&c.branch);
13242 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13243 if !git::branch_delete(&repo, &c.branch).await? {
13244 bail!("candidate branch removal refused");
13245 }
13246 removed.push(c.branch.clone());
13247 }
13248 state.candidates[i].folded = true;
13249 }
13250
13251 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13252 let path = name.path();
13253 let keep = !drop_winner
13254 && winner.is_some_and(|w| {
13255 path.file_name()
13256 .is_some_and(|n| n == format!("cand-{w}").as_str())
13257 });
13258 if keep {
13259 continue;
13260 }
13261 remove_preserved_worktree(&repo, &path).await?;
13262 removed.push(path.to_string_lossy().into_owned());
13263 }
13264
13265 remove_if_empty(&root);
13274
13275 if state.enabled_worktree_config && drop_winner {
13276 git::release_worktree_config(&repo).await.ok();
13280 state.enabled_worktree_config = false;
13281 }
13282 state.save_under(home)?;
13283 Ok(removed)
13284}
13285
13286async fn verify_merged_fold_sources(
13287 state: &RunState,
13288 drop_winner: bool,
13289 home: &Path,
13290) -> Result<Vec<std::fs::File>> {
13291 let winner = state.tally.as_ref().map(|t| t.winner);
13292 let mut heads = BTreeSet::new();
13293 for candidate in &state.candidates {
13294 if Some(candidate.label) == winner && !drop_winner {
13295 continue;
13296 }
13297 anyhow::ensure!(
13298 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13299 "candidate recovery source is outside the inventoried run root: {}",
13300 candidate.worktree.display()
13301 );
13302 match std::fs::symlink_metadata(&candidate.worktree) {
13303 Ok(_) => {}
13304 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13305 Err(error) => return Err(error.into()),
13306 }
13307 if !state.released_branches.contains(&candidate.branch)
13308 && git::branch_exists(&state.repo, &candidate.branch).await?
13309 {
13310 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13311 }
13312 }
13313 let mut cargo_guards = Vec::new();
13314 let entries = match std::fs::read_dir(state.worktree_root()) {
13315 Ok(entries) => Some(entries),
13316 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13317 Err(error) => return Err(error.into()),
13318 };
13319 for entry in entries.into_iter().flatten() {
13320 let path = entry?.path();
13321 let metadata = std::fs::symlink_metadata(&path)?;
13322 anyhow::ensure!(
13323 metadata.is_dir() && !metadata.file_type().is_symlink(),
13324 "non-directory or symlink recovery source retained: {}",
13325 path.display()
13326 );
13327 if !drop_winner
13328 && winner.is_some_and(|w| {
13329 path.file_name()
13330 .is_some_and(|name| name == format!("cand-{w}").as_str())
13331 })
13332 {
13333 continue;
13334 }
13335 anyhow::ensure!(
13336 path.join(".git").is_file(),
13337 "unregistered recovery source retained: {}",
13338 path.display()
13339 );
13340 anyhow::ensure!(
13341 git::git(
13342 &path,
13343 &[
13344 "status",
13345 "--porcelain",
13346 "--untracked-files=all",
13347 "--ignored",
13348 "--",
13349 ".",
13350 ":(exclude)target"
13351 ]
13352 )
13353 .await?
13354 .is_empty(),
13355 "source changes or ignored recovery files retained: {}",
13356 path.display()
13357 );
13358 match std::fs::symlink_metadata(path.join("target")) {
13359 Ok(_) => {
13360 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13361 }
13362 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13363 Err(error) => return Err(error.into()),
13364 }
13365 for checkpoint in [
13366 "rebase-merge",
13367 "rebase-apply",
13368 "sequencer",
13369 "CHERRY_PICK_HEAD",
13370 "MERGE_HEAD",
13371 "REBASE_HEAD",
13372 "REVERT_HEAD",
13373 "BISECT_LOG",
13374 "AUTO_MERGE",
13375 "MERGE_MSG",
13376 ] {
13377 let location = git::git(
13378 &path,
13379 &[
13380 "rev-parse",
13381 "--path-format=absolute",
13382 "--git-path",
13383 checkpoint,
13384 ],
13385 )
13386 .await?;
13387 anyhow::ensure!(
13388 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13389 if e.kind() == std::io::ErrorKind::NotFound),
13390 "Git recovery checkpoint retained: {checkpoint}"
13391 );
13392 }
13393 heads.insert(git::rev_parse(&path, "HEAD").await?);
13394 }
13395 crate::fold_evidence::preserve(state, home, &heads).await?;
13396
13397 Ok(cargo_guards)
13398}
13399
13400fn remove_if_empty(dir: &Path) {
13411 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13412 std::fs::remove_dir(dir).ok();
13413 }
13414}
13415
13416pub fn worst_open(state: &RunState) -> Option<Severity> {
13418 state
13419 .reviews
13420 .last()?
13421 .reviews
13422 .iter()
13423 .flat_map(|r| r.findings.iter())
13424 .map(|f| f.severity)
13425 .max()
13426}
13427
13428async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13432 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13433 git::acquire_worktree_config(repo).await?;
13434 state.enabled_worktree_config = true;
13435 }
13436 Ok(())
13437}
13438
13439#[cfg(test)]
13440mod tests {
13441 #[tokio::test]
13442 async fn launcher_base_fetch_requires_remote_success() {
13443 let tmp = tempfile::tempdir().unwrap();
13444 let repo = tmp.path().join("repo");
13445 let remote = tmp.path().join("remote");
13446 std::fs::create_dir(&repo).unwrap();
13447 std::fs::create_dir(&remote).unwrap();
13448 git::git(&remote, &["init", "--bare", "-b", "main"])
13449 .await
13450 .unwrap();
13451 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13452 git::git(
13453 &repo,
13454 &[
13455 "-c",
13456 "user.name=Fixture",
13457 "-c",
13458 "user.email=fixture@example.invalid",
13459 "commit",
13460 "--allow-empty",
13461 "-m",
13462 "fixture",
13463 ],
13464 )
13465 .await
13466 .unwrap();
13467 git::git(
13468 &repo,
13469 &["remote", "add", "origin", remote.to_str().unwrap()],
13470 )
13471 .await
13472 .unwrap();
13473 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13474 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13475 assert_eq!(
13476 resolve_base(&repo, "main", "origin", None)
13477 .await
13478 .unwrap()
13479 .commit,
13480 head
13481 );
13482 #[cfg(unix)]
13483 {
13484 let policy: crate::config::Coordination = toml::from_str(
13485 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13486 ).unwrap();
13487 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13488 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13489 let held = crate::repo_lock::SeatLease::operation_with(
13490 &repo,
13491 Some(&policy),
13492 &crate::repo_lock::WaitPolicy::default(),
13493 )
13494 .await
13495 .unwrap();
13496 assert!(held.is_some());
13497 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13498 tokio::pin!(fetch);
13499 assert!(
13500 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13501 .await
13502 .is_err()
13503 );
13504 assert!(
13505 !repo.join(".git/FETCH_HEAD").exists(),
13506 "no fetch before admission"
13507 );
13508 drop(held);
13509 let fetched = fetch.await.unwrap();
13510 assert_eq!(fetched.commit, head);
13511 assert_eq!(fetched.remote, "origin");
13512 assert_eq!(fetched.branch, "main");
13513 }
13514 git::git(
13516 &repo,
13517 &[
13518 "remote",
13519 "set-url",
13520 "origin",
13521 tmp.path().join("missing").to_str().unwrap(),
13522 ],
13523 )
13524 .await
13525 .unwrap();
13526 let error = resolve_base(&repo, "main", "origin", None)
13527 .await
13528 .unwrap_err()
13529 .to_string();
13530 assert!(error.contains("cannot read origin/main"));
13531 assert!(error.contains("refusing to branch off"));
13532 }
13533
13534 #[test]
13535 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13536 let leaked = vec!["chore(deps): update a crate".to_owned()];
13537 let own = "fix(daemon): apply a chosen action";
13538 assert!(should_retitle("", own, &leaked));
13539 assert!(should_retitle(
13540 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13541 own,
13542 &leaked
13543 ));
13544 assert!(should_retitle(
13545 "chore: land candidate A of run 1",
13546 own,
13547 &leaked
13548 ));
13549 assert!(should_retitle(
13550 "magi: candidate A (uncommitted work)",
13551 own,
13552 &leaked
13553 ));
13554 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13555 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13556 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13557 }
13558
13559 #[test]
13560 fn pr_merge_plan_creates_adopts_or_stops() {
13561 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13562 assert_eq!(
13563 pr_merge_plan(Ok(land::OpenPr::One {
13564 url: "u".into(),
13565 title: "t".into()
13566 })),
13567 PrPlan::Adopt {
13568 url: "u".into(),
13569 title: "t".into()
13570 }
13571 );
13572 let PrPlan::Stop(many) =
13573 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13574 else {
13575 panic!("many must stop");
13576 };
13577 assert!(many.contains('a') && many.contains('b'));
13578 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13579 panic!("a failed lookup must stop");
13580 };
13581 assert!(err.contains("bad token"));
13582 }
13583
13584 use super::*;
13585 use crate::run::GateStatus;
13586 use std::collections::BTreeMap;
13587 use std::time::Duration;
13588
13589 fn conductor() -> AgentSpec {
13590 AgentSpec {
13591 id: "conductor".to_owned(),
13592 kind: crate::config::AgentKind::Command,
13593 model: None,
13594 command: vec!["true".to_owned()],
13595 extra_args: Vec::new(),
13596 env: BTreeMap::new(),
13597 prompt_delivery: None,
13598 max_turn_secs: None,
13599 }
13600 }
13601
13602 fn spec(id: &str) -> AgentSpec {
13603 AgentSpec {
13604 id: id.to_owned(),
13605 kind: crate::config::AgentKind::Command,
13606 model: None,
13607 command: vec!["true".to_owned()],
13608 extra_args: Vec::new(),
13609 env: BTreeMap::new(),
13610 prompt_delivery: None,
13611 max_turn_secs: None,
13612 }
13613 }
13614
13615 fn ids(xs: &[&str]) -> BTreeSet<String> {
13616 xs.iter().map(|s| (*s).to_owned()).collect()
13617 }
13618
13619 #[test]
13620 fn pick_successor_skips_an_agent_another_seat_holds() {
13621 let roster = [spec("a"), spec("b"), spec("c")];
13623 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13624 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13625 }
13626
13627 #[test]
13628 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13629 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13631 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13632 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13633 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13634 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13635 }
13636
13637 #[test]
13638 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13639 let roster = [spec("a"), spec("b")];
13640 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13641 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13642 let carried = ids(&["b"]);
13643 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13644 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13645 }
13646
13647 #[test]
13648 fn pick_successor_returns_none_without_any_untried_successor() {
13649 let roster = [spec("a"), spec("b")];
13650 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13651 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13652 let carried = ids(&["a"]);
13653 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13654 }
13655
13656 #[test]
13657 fn pick_successor_rescue_avoids_another_seats_occupant() {
13658 let roster = [spec("a"), spec("b"), spec("c")];
13660 let carried = ids(&["b", "c"]);
13661 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13662 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13663 }
13664
13665 #[test]
13666 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13667 let roster = [spec("a"), spec("b"), spec("c")];
13668 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13669 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13670 }
13671
13672 #[test]
13673 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13674 let roster = [spec("a"), spec("b"), spec("c")];
13675 let failed = ids(&["a", "b", "c"]);
13676 let mut tried = ids(&["b"]);
13678 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13679 assert_eq!(first.id, "a");
13680 tried.insert(first.id.clone());
13681 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13682 assert_eq!(second.id, "c");
13683 tried.insert(second.id.clone());
13684 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13685 }
13686
13687 #[test]
13688 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13689 let roster = [spec("a"), spec("b")];
13690 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13691 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13692 }
13693
13694 #[test]
13695 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13696 let roster = [spec("a"), spec("b"), spec("c")];
13697 let h = SeatHistory {
13698 failed: ids(&["a"]),
13699 last_ok: Some("b".to_owned()),
13700 last_fail: None,
13701 };
13702 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13703 let gone = SeatHistory {
13705 last_ok: Some("zzz".to_owned()),
13706 ..h.clone()
13707 };
13708 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13709 let failed = SeatHistory {
13710 failed: ids(&["a", "b"]),
13711 last_ok: Some("b".to_owned()),
13712 last_fail: None,
13713 };
13714 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13715 }
13716
13717 #[test]
13718 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13719 let first = SeatState::new("review-1", "alpha", 7);
13720 let next = handover_seat("review-1", "gamma", 7);
13721 assert_ne!(first.claude_session, next.claude_session);
13722 }
13723
13724 #[test]
13725 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13726 let mut state = state_with_summary("x", "y");
13727 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13728 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13729 assert_ne!(a.claude_session, b.claude_session);
13730 }
13731
13732 #[test]
13733 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13734 let roster = [spec("a"), spec("b")];
13735 let h = SeatHistory {
13736 failed: ids(&["a", "b"]),
13737 ..SeatHistory::default()
13738 };
13739 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13740 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13741 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13742 }
13743
13744 #[test]
13751 fn failure_signature_ignores_numbers_and_paths() {
13752 assert_eq!(
13753 failure_signature("exited with Some(2) and no usable output"),
13754 failure_signature("exited with Some(137) and no usable output")
13755 );
13756 assert_eq!(
13757 failure_signature("process exited with no usable output (exit code 2)"),
13758 failure_signature("process exited with no usable output (exit code 137)")
13759 );
13760 for (old, new) in [
13761 (
13762 "exited with Some(2) and no usable output",
13763 "process exited with no usable output (exit code 2)",
13764 ),
13765 (
13766 "exited with None and no usable output",
13767 "process exited with no usable output (exit code unavailable)",
13768 ),
13769 ] {
13770 let previous = FailClass::Other(failure_signature(old));
13771 let current = FailClass::Other(failure_signature(new));
13772 assert_eq!(previous, current, "resumption retains the failure class");
13773 assert!(!should_hand_over(Some(&previous), ¤t));
13774 }
13775 assert_eq!(
13776 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13777 failure_signature("cannot open /var/x.txt: denied")
13778 );
13779 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13780 }
13781
13782 #[test]
13783 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13784 let other = FailClass::Other("x".into());
13785 assert!(should_hand_over(None, &FailClass::Quota));
13786 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13787 assert!(should_hand_over(
13788 Some(&FailClass::Timeout),
13789 &FailClass::Timeout
13790 ));
13791 assert!(should_hand_over(None, &other));
13792 assert!(!should_hand_over(Some(&other), &other));
13793 assert!(should_hand_over(
13794 Some(&other),
13795 &FailClass::Other("y".into())
13796 ));
13797 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13799 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13800 }
13801
13802 #[test]
13803 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13804 let a = SeatState::new("judge-1", "alpha", 7);
13805 let b = handover_seat("judge-1", "beta", 7);
13806 assert_ne!(a.claude_session, b.claude_session);
13807 assert_eq!(b.turns, 0);
13808 }
13809
13810 #[test]
13811 fn a_timeout_is_classified_apart_from_other_failures() {
13812 assert_eq!(
13813 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13814 Some(FailClass::Timeout)
13815 );
13816 assert!(matches!(
13817 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13818 Some(FailClass::Other(_))
13819 ));
13820 }
13821
13822 #[test]
13823 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13824 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13825 let tried = BTreeSet::from(["beta".to_owned()]);
13826 let next = next_untried_in_roster(&roster, 1, &tried);
13829 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13830 }
13831
13832 #[test]
13833 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13834 let roster = vec![spec("alpha"), spec("beta")];
13835 let tried = BTreeSet::from(["beta".to_owned()]);
13836 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13840 }
13841
13842 #[test]
13843 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13844 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13845 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13846 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13850 }
13851
13852 #[test]
13853 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13854 let roster = vec![spec("a"), spec("a"), spec("b")];
13855 let tried = BTreeSet::from(["a".to_owned()]);
13856 let next = next_untried_in_roster(&roster, 0, &tried);
13857 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13858 }
13859
13860 #[test]
13861 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13862 let roster = vec![spec("a"), spec("b")];
13863 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13864 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13865 }
13866
13867 #[test]
13868 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13869 let dir = tempfile::tempdir().unwrap();
13870 let bay = dir.path().join("ffff");
13871
13872 remove_if_empty(&bay);
13874 assert!(!bay.exists());
13875
13876 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13879 remove_if_empty(&bay);
13880 assert!(bay.exists(), "non-empty directory must survive");
13881
13882 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13884 remove_if_empty(&bay);
13885 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13886 }
13887
13888 #[test]
13897 fn a_full_panel_that_found_nothing_is_clean() {
13898 assert!(round_is_clean(
13899 0,
13900 true,
13901 2,
13902 2,
13903 0,
13904 IncompleteReviewPolicy::Block
13905 ));
13906 }
13907
13908 #[test]
13909 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13910 assert!(!round_is_clean(
13911 0,
13912 true,
13913 1,
13914 2,
13915 0,
13916 IncompleteReviewPolicy::Block
13917 ));
13918 }
13919
13920 #[test]
13921 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13922 assert!(!round_is_clean(
13923 1,
13924 true,
13925 1,
13926 2,
13927 0,
13928 IncompleteReviewPolicy::Warn
13929 ));
13930 }
13931
13932 #[test]
13933 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13934 assert!(round_is_clean(
13935 0,
13936 true,
13937 1,
13938 2,
13939 0,
13940 IncompleteReviewPolicy::Warn
13941 ));
13942 }
13943
13944 #[test]
13945 fn a_full_panel_with_an_open_finding_is_not_clean() {
13946 assert!(!round_is_clean(
13947 1,
13948 true,
13949 2,
13950 2,
13951 0,
13952 IncompleteReviewPolicy::Block
13953 ));
13954 }
13955
13956 #[test]
13957 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13958 assert!(!round_is_clean(
13959 0,
13960 false,
13961 2,
13962 2,
13963 0,
13964 IncompleteReviewPolicy::Block
13965 ));
13966 }
13967
13968 #[test]
13975 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13976 assert!(round_is_clean(
13979 0,
13980 true,
13981 1,
13982 2,
13983 1,
13984 IncompleteReviewPolicy::Block
13985 ));
13986 }
13987
13988 #[test]
13989 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13990 assert!(!round_is_clean(
13993 0,
13994 true,
13995 1,
13996 2,
13997 0,
13998 IncompleteReviewPolicy::Block
13999 ));
14000 }
14001
14002 #[test]
14003 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14004 assert!(!round_is_clean(
14005 1,
14006 true,
14007 1,
14008 2,
14009 1,
14010 IncompleteReviewPolicy::Block
14011 ));
14012 assert!(!round_is_clean(
14013 0,
14014 false,
14015 1,
14016 2,
14017 1,
14018 IncompleteReviewPolicy::Block
14019 ));
14020 }
14021
14022 #[test]
14023 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14024 assert!(!round_is_clean(
14028 0,
14029 true,
14030 0,
14031 2,
14032 2,
14033 IncompleteReviewPolicy::Block
14034 ));
14035 }
14036
14037 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14038 CommandOutcome {
14039 command: "test".to_owned(),
14040 code,
14041 output_tail: String::new(),
14042 duration_ms: 0,
14043 resource_blocked,
14044 }
14045 }
14046
14047 #[cfg(unix)]
14048 #[tokio::test]
14049 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14050 ask_test_home();
14051 let temp = tempfile::tempdir().unwrap();
14052 let mut runner = runner_at(RunStatus::Gating);
14053 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14054 let commands = vec![
14055 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14056 "touch must-not-run".to_owned(),
14057 ];
14058 let (outcomes, retried, pids) = run_e2e_with_retry(
14059 &mut runner.state,
14060 &shell,
14061 &commands,
14062 temp.path(),
14063 Duration::from_secs(5),
14064 "fixture",
14065 )
14066 .await;
14067 assert_eq!(outcomes.len(), 1);
14068 assert!(outcomes[0].resource_blocked);
14069 assert!(!outcomes[0].build_failed());
14070 assert!(!retried);
14071 assert!(pids.is_empty());
14072 assert!(!temp.path().join("must-not-run").exists());
14073 assert_eq!(
14074 runner.state.resource_block.as_ref().unwrap().kind,
14075 "incremental cache admission"
14076 );
14077 }
14078
14079 #[test]
14080 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14081 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14082 assert!(
14083 !verify_inconclusive(&[outcome(Some(1), false)]),
14084 "an ordinary failure is still evidence about the patch"
14085 );
14086 assert!(verify_inconclusive(&[outcome(None, true)]));
14087 assert!(
14088 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14089 "one inconclusive outcome taints the whole batch"
14090 );
14091 assert!(!verify_inconclusive(&[]));
14092 }
14093
14094 #[tokio::test]
14095 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14096 let calls = std::sync::atomic::AtomicUsize::new(0);
14100 let started = Instant::now();
14101 wait_for_pids_with(
14102 &[123],
14103 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14104 Duration::from_millis(5),
14105 Duration::from_secs(5),
14106 )
14107 .await;
14108 assert!(
14109 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14110 "must keep checking rather than deciding on the first answer"
14111 );
14112 assert!(
14113 started.elapsed() < Duration::from_secs(1),
14114 "must return the moment it is confirmed dead, not wait out the ceiling"
14115 );
14116 }
14117
14118 #[tokio::test]
14119 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14120 let started = Instant::now();
14121 wait_for_pids_with(
14122 &[123],
14123 |_| true, Duration::from_millis(5),
14125 Duration::from_millis(30),
14126 )
14127 .await;
14128 let elapsed = started.elapsed();
14129 assert!(
14130 elapsed >= Duration::from_millis(30),
14131 "must not give up before its own ceiling: {elapsed:?}"
14132 );
14133 assert!(
14134 elapsed < Duration::from_secs(1),
14135 "must not wait past its own ceiling either: {elapsed:?}"
14136 );
14137 }
14138
14139 #[tokio::test]
14140 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14141 let started = Instant::now();
14142 wait_for_pids_with(
14143 &[],
14144 |_| true,
14145 Duration::from_secs(5),
14146 Duration::from_secs(5),
14147 )
14148 .await;
14149 assert!(
14150 started.elapsed() < Duration::from_millis(200),
14151 "an empty pid list has nothing to confirm"
14152 );
14153 }
14154
14155 fn review_round(
14161 clean: bool,
14162 blocking: usize,
14163 answered: usize,
14164 expected: usize,
14165 progressed: bool,
14166 e2e_ok: bool,
14167 ) -> ReviewRound {
14168 ReviewRound {
14169 round: 1,
14170 head: "h".to_owned(),
14171 verified_head: None,
14172 verified_at: None,
14173 reviews: Vec::new(),
14174 e2e: vec![CommandOutcome {
14175 command: "test".to_owned(),
14176 code: Some(if e2e_ok { 0 } else { 1 }),
14177 output_tail: String::new(),
14178 duration_ms: 0,
14179 resource_blocked: false,
14180 }],
14181 verify_retried: false,
14182 e2e_deferred: false,
14183 e2e_defer_reason: None,
14184 fix: None,
14185 blocking,
14186 answered,
14187 expected,
14188 clean,
14189 progressed,
14190 vote_split: false,
14191 reconsideration: Vec::new(),
14192 verdict: None,
14193 }
14194 }
14195
14196 #[test]
14197 fn review_conclusion_is_none_when_nothing_has_run() {
14198 assert_eq!(review_conclusion(&[], 3), None);
14199 }
14200
14201 #[test]
14202 fn review_conclusion_is_none_while_rounds_remain() {
14203 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14204 assert_eq!(review_conclusion(&rounds, 3), None);
14205 }
14206
14207 #[test]
14208 fn review_conclusion_is_gating_once_a_round_is_clean() {
14209 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14210 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14211 }
14212
14213 #[test]
14214 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14215 let rounds = vec![
14216 review_round(false, 1, 2, 2, true, true),
14217 review_round(false, 1, 2, 2, true, true),
14218 ];
14219 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14220 }
14221
14222 #[test]
14223 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14224 let rounds = vec![
14225 review_round(false, 1, 2, 2, true, true),
14226 review_round(false, 1, 2, 2, true, false),
14227 ];
14228 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14229 }
14230
14231 #[test]
14232 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14233 let mut blocked = review_round(false, 1, 2, 2, true, false);
14240 blocked.e2e[0].resource_blocked = true;
14241 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14242 assert_eq!(review_conclusion(&rounds, 2), None);
14243 }
14244
14245 #[test]
14246 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14247 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14249 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14250 }
14251
14252 #[test]
14253 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14254 let rounds = vec![
14255 review_round(false, 1, 2, 2, false, true),
14256 review_round(false, 1, 2, 2, false, true),
14257 ];
14258 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14259 }
14260
14261 fn secs(n: u64) -> Duration {
14262 Duration::from_secs(n)
14263 }
14264
14265 #[tokio::test]
14268 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14269 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14270 if std::env::var_os(CHILD).is_none() {
14271 let output = std::process::Command::new(std::env::current_exe().unwrap())
14275 .args([
14276 "--exact",
14277 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14278 "--nocapture",
14279 ])
14280 .env(CHILD, "1")
14281 .env_remove("MAGI_WORKER_HOME")
14282 .env_remove("MAGI_WORKER_ID")
14283 .output()
14284 .unwrap();
14285 assert!(
14286 output.status.success()
14287 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14288 "{}\n{}",
14289 String::from_utf8_lossy(&output.stdout),
14290 String::from_utf8_lossy(&output.stderr)
14291 );
14292 return;
14293 }
14294 ask_test_home();
14295 let tmp = tempfile::tempdir().unwrap();
14296 let repo = tmp.path().join("repo");
14297 std::fs::create_dir_all(&repo).unwrap();
14298 init_repo(&repo);
14299 let g = |args: &[&str]| {
14300 let out = std::process::Command::new("git")
14301 .args(args)
14302 .current_dir(&repo)
14303 .quiet()
14304 .output()
14305 .unwrap();
14306 assert!(
14307 out.status.success(),
14308 "{}",
14309 String::from_utf8_lossy(&out.stderr)
14310 );
14311 };
14312 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14313 g(&["checkout", "-b", "historical"]);
14314 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14315 g(&["add", "."]);
14316 g(&["commit", "-m", "old alternative"]);
14317 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14318 g(&["checkout", "-b", "magi/task-selected", &base]);
14319 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14320 g(&["add", "."]);
14321 g(&["commit", "-m", "measurement infrastructure"]);
14322 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14323 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14324 g(&["add", "."]);
14325 g(&["commit", "-m", "dependent implementation"]);
14326 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14327 g(&["checkout", "main"]);
14328 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14329 g(&["add", "."]);
14330 g(&["commit", "-m", "unrelated base change"]);
14331 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14332 let origin = tmp.path().join("origin.git");
14333 g(&[
14334 "clone",
14335 "--bare",
14336 repo.to_str().unwrap(),
14337 origin.to_str().unwrap(),
14338 ]);
14339 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14340 g(&["fetch", "origin"]);
14341 let selection = crate::pr_recovery::Selection {
14342 url: "https://example.test/project/pull/1".into(),
14343 branch: "magi/task-selected".into(),
14344 head: head.clone(),
14345 source_head: None,
14346 source_run: None,
14347 source_proof: None,
14348 disposition: Some(crate::pr_recovery::Disposition::Review),
14349 instruction: "preserve measured behavior".into(),
14350 approved_at: jiff::Timestamp::now(),
14351 };
14352 let mut template = runner_at(RunStatus::Reviewing);
14353 template.state.repo = repo.clone();
14354 template.state.base_commit = current.clone();
14355 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14356 template.state.origin = Some(crate::run::Origin::queue("selected"));
14357 template.state.pr_head = Some(selection.branch.clone());
14358 template.state.pr_recovery = Some(selection.clone());
14359 let mut runner =
14360 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14361 .await
14362 .unwrap();
14363 let wt = runner.state.winner().unwrap().worktree.clone();
14364 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14365 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14366 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14367 assert!(runner.state.seeds.is_empty());
14368 assert_eq!(runner.state.instruction, "preserve measured behavior");
14369 runner.sync_to_base().await.unwrap();
14370 assert_eq!(
14371 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14372 "prerequisite"
14373 );
14374 assert_eq!(
14375 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14376 "measured implementation"
14377 );
14378 assert_eq!(
14379 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14380 "current main change"
14381 );
14382 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14383 assert_eq!(
14384 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14385 head
14386 );
14387
14388 let mut partial = runner_at(RunStatus::Reviewing);
14390 partial.state.repo = repo.clone();
14391 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14392 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14393 run: Some("exit 1".into()),
14394 ..Default::default()
14395 }];
14396 let selection = runner.state.pr_recovery.clone().unwrap();
14397 partial.state.pr_recovery = Some(selection.clone());
14398 partial.state.pr_head = Some(selection.branch.clone());
14399 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14400 let id = partial.state.id.clone();
14401 let error = match Runner::open_selected(
14402 &repo,
14403 partial.state,
14404 partial.roles,
14405 1,
14406 runner.state.base_commit.clone(),
14407 selection,
14408 )
14409 .await
14410 {
14411 Ok(_) => panic!("setup must fail"),
14412 Err(e) => e,
14413 };
14414 assert_eq!(
14415 error
14416 .downcast_ref::<crate::run::StartupFailure>()
14417 .unwrap()
14418 .run,
14419 id
14420 );
14421 let failed = RunState::load(&id).unwrap();
14422 assert_eq!(failed.status, RunStatus::Failed);
14423 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14424 assert!(Runner::resume(&id).is_err());
14425 runner.state.status = RunStatus::Ready;
14426 runner.state.driver_exited = true;
14427 runner.state.save().unwrap();
14428 let mut retry = runner_at(RunStatus::Reviewing);
14429 retry.state.repo = repo.clone();
14430 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14431 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14432 reselected.approved_at = jiff::Timestamp::now();
14433 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14434 retry.state.pr_head = Some(reselected.branch.clone());
14435 retry.state.pr_recovery = Some(reselected.clone());
14436 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14437 .await
14438 .unwrap();
14439 let restarted = Runner::open_selected(
14440 &repo,
14441 retry.state,
14442 retry.roles,
14443 1,
14444 runner.state.base_commit.clone(),
14445 reselected,
14446 )
14447 .await
14448 .unwrap();
14449 assert_eq!(
14450 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14451 .await
14452 .unwrap(),
14453 head
14454 );
14455 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14456 assert_eq!(
14457 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14458 head
14459 );
14460 }
14461
14462 #[tokio::test]
14463 #[cfg(unix)]
14464 async fn fallback_and_interrupted_restoration_require_coordination() {
14465 use std::os::fd::AsRawFd as _;
14466 ask_test_home();
14467 let tmp = tempfile::tempdir().unwrap();
14468 let mut runner = registered_runner(tmp.path()).await;
14469 runner.state.config.graph.timeout_resource_wait = 1;
14470 let winner = runner.state.candidates[0].clone();
14471 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14472 let leftover = winner.worktree.join("leftover");
14473 std::fs::write(&leftover, "untrusted edits").unwrap();
14474 let next = tmp.path().join("next-fixer");
14475 let mut failed = spec("first");
14476 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14477 let mut successor = spec("second");
14478 successor.command = vec![
14479 "sh".into(),
14480 "-c".into(),
14481 format!("touch '{}'; echo done", next.display()),
14482 ];
14483 let attempts = vec![
14484 (failed.clone(), "first".into()),
14485 (successor.clone(), "second".into()),
14486 ];
14487 let lock_path = runner.state.repo.join(".git/updater.lock");
14488 let external = std::fs::File::create(&lock_path).unwrap();
14489 assert_eq!(
14490 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14491 0
14492 );
14493 let artifacts = tmp.path().join("artifacts");
14494 let build = |spec, seat| SeatJob {
14495 spec,
14496 seat,
14497 cwd: winner.worktree.clone(),
14498 prompt: "fixture".into(),
14499 timeout: Duration::from_secs(10),
14500 allow_write: true,
14501 sessions: false,
14502 artifacts: artifacts.clone(),
14503 stem: "fix".into(),
14504 handover: None,
14505 };
14506 let (_, _, out) = runner
14507 .ask_fixer_among(
14508 attempts.clone(),
14509 "graft",
14510 None,
14511 Some((&winner.worktree, &head)),
14512 build,
14513 )
14514 .await;
14515 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14516 assert!(leftover.exists(), "contention must prevent clean");
14517 assert!(!next.exists());
14518 assert!(runner.state.handovers.is_empty());
14519 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14521 from_run: "earlier".into(),
14522 generation: 1,
14523 mode: crate::recovery::Mode::VerifyRepair,
14524 reasons: Vec::new(),
14525 reviewed_head: None,
14526 rounds: Vec::new(),
14527 open_findings: Vec::new(),
14528 evidence: None,
14529 criteria_inherited: true,
14530 repair: crate::recovery::Repair::Started {
14531 before: head.clone(),
14532 },
14533 });
14534 assert!(
14535 runner
14536 .recovery_repair(&winner)
14537 .await
14538 .unwrap_err()
14539 .is::<crate::repo_lock::ResourceBlocked>()
14540 );
14541 assert!(matches!(
14542 runner.state.recovery.as_ref().unwrap().repair,
14543 crate::recovery::Repair::Started { .. }
14544 ));
14545 assert!(leftover.exists());
14546 drop(external);
14547 runner.recovery_repair(&winner).await.unwrap();
14548 assert!(!leftover.exists());
14549 assert!(matches!(
14550 runner.state.recovery.as_ref().unwrap().repair,
14551 crate::recovery::Repair::Done { .. }
14552 ));
14553 std::fs::remove_file(&lock_path).unwrap();
14555 std::fs::create_dir(&lock_path).unwrap();
14556 std::fs::write(&leftover, "untrusted edits").unwrap();
14557 let (_, _, out) = runner
14558 .ask_fixer_among(
14559 attempts,
14560 "graft",
14561 None,
14562 Some((&winner.worktree, &head)),
14563 build,
14564 )
14565 .await;
14566 assert!(matches!(out, AgentOutcome::Failed(_)));
14567 assert!(leftover.exists());
14568 assert!(!next.exists());
14569 assert_eq!(
14570 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14571 head
14572 );
14573 }
14574
14575 #[cfg(unix)]
14576 async fn registered_runner(root: &Path) -> Runner {
14577 repository_runner(root, crate::config::test_coordination()).await
14578 }
14579
14580 async fn repository_runner(
14581 root: &Path,
14582 coordination: Option<crate::config::Coordination>,
14583 ) -> Runner {
14584 ask_test_home();
14585 let repo = root.join("repo");
14586 std::fs::create_dir_all(&repo).unwrap();
14587 init_repo(&repo);
14588 if coordination.is_some() {
14589 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14590 }
14591 let mut runner = runner_at(RunStatus::Prep);
14592 runner.state.repo = repo.clone();
14593 runner.state.config.coordination = coordination;
14594 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14595 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14596 runner.state.config.graph.advise = false;
14597 runner.roles.implementers = vec![spec("implement")];
14598 runner.roles.judges = vec![spec("judge")];
14599 runner.prep().await.unwrap();
14600 runner
14601 }
14602
14603 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14604
14605 #[tokio::test]
14606 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14607 let tmp = tempfile::tempdir().unwrap();
14608 let mut runner = repository_runner(tmp.path(), None).await;
14609 let prompts = runner.state.config.prompts.clone();
14610 let run = runner.state.id.clone();
14611 let ctx = WaveCtx {
14612 carry_seats: false,
14613 pauses: None,
14614 run: &run,
14615 node: "advise",
14616 prompts: &prompts,
14617 cache: None,
14618 round: None,
14619 };
14620 let jobs: Vec<_> = ["alpha", "beta"]
14621 .into_iter()
14622 .map(|id| SeatJob {
14623 spec: scripted(id, AUTH_OUTAGE),
14624 seat: SeatState::new(id, id, 7),
14625 prompt: "propose".into(),
14626 cwd: runner.state.candidates[0].worktree.clone(),
14627 timeout: Duration::from_secs(10),
14628 allow_write: false,
14629 sessions: false,
14630 artifacts: agent::artifacts_dir(&runner.state.dir()),
14631 stem: id.into(),
14632 handover: None,
14633 })
14634 .collect();
14635 let mut losses = Vec::new();
14636 let results = ask_wave_with(
14637 jobs.clone(),
14638 runner.sem.clone(),
14639 2,
14640 &[],
14641 &ctx,
14642 &mut losses,
14643 &mut runner.state,
14644 &|_: &str| Ok(()),
14645 )
14646 .await;
14647 assert!(
14648 results
14649 .iter()
14650 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14651 );
14652 assert_eq!(runner.state.quota.len(), 2);
14653 assert!(
14654 runner
14655 .state
14656 .quota
14657 .iter()
14658 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14659 );
14660 assert!(!runner.state.advise_attempted);
14661 runner.stop_for_resource().unwrap();
14662 assert_eq!(runner.state.status, RunStatus::Blocked);
14663 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14664 assert_eq!(
14665 runner.state.resource_block.as_ref().unwrap().kind,
14666 "agent authentication"
14667 );
14668 runner.state.resource_block = None;
14671 runner.state.quota.clear();
14672 let mut mixed = jobs;
14673 mixed[1].spec.command = vec!["false".into()];
14674 let results = ask_wave_with(
14675 mixed,
14676 runner.sem.clone(),
14677 0,
14678 &[],
14679 &ctx,
14680 &mut losses,
14681 &mut runner.state,
14682 &|_: &str| Ok(()),
14683 )
14684 .await;
14685 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14686 assert!(runner.state.resource_block.is_none());
14687 assert_eq!(losses.len(), 1);
14688 }
14689
14690 #[tokio::test]
14691 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14692 let tmp = tempfile::tempdir().unwrap();
14693 let mut runner = repository_runner(tmp.path(), None).await;
14694 let implementer = scripted("implement", AUTH_OUTAGE);
14695 runner.roles.implementers = vec![implementer.clone()];
14696 runner.state.config.agents = vec![implementer];
14697 runner.implement().await.unwrap();
14698 assert_eq!(runner.state.status, RunStatus::Blocked);
14699 assert_eq!(runner.state.quota.len(), 1);
14700 assert_eq!(
14701 runner.state.quota[0].cause,
14702 agent::QuotaCause::Authentication
14703 );
14704 let candidate = &runner.state.candidates[0];
14705 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14706 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14707 }
14708
14709 #[tokio::test]
14710 #[cfg(unix)]
14711 async fn parked_implementer_persists_completed_work() {
14712 for registered in [false, true] {
14713 let tmp = tempfile::tempdir().unwrap();
14714 let mut runner = registered_runner(tmp.path()).await;
14715 if !registered {
14716 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14717 }
14718 let marker = tmp.path().join("started");
14719 let release = tmp.path().join("release");
14720 let mut implementer = spec("implement");
14721 implementer.command = vec![
14722 "sh".into(),
14723 "-c".into(),
14724 format!(
14725 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14726 marker.display(),
14727 release.display()
14728 ),
14729 ];
14730 runner.roles.implementers = vec![implementer.clone()];
14731 runner.state.config.agents = vec![implementer];
14732 let pause = runner.pause.clone();
14733 let (result, ()) = tokio::join!(runner.implement(), async {
14734 tokio::time::timeout(Duration::from_secs(10), async {
14735 while !marker.exists() {
14736 tokio::time::sleep(Duration::from_millis(10)).await;
14737 }
14738 })
14739 .await
14740 .unwrap();
14741 pause.park();
14742 std::fs::write(&release, "go").unwrap();
14743 });
14744 result.unwrap();
14745 let candidate = &runner.state.candidates[0];
14746 assert!(candidate.commits > 0, "{candidate:?}");
14747 assert!(candidate.summary.contains("saved-result"));
14748 assert_eq!(candidate.files, 1);
14749 let before = candidate.commits;
14750 runner.implement().await.unwrap();
14751 assert_eq!(runner.state.candidates[0].commits, before);
14752 }
14753 }
14754
14755 #[tokio::test]
14756 #[cfg(unix)]
14757 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14758 let tmp = tempfile::tempdir().unwrap();
14759 let mut runner = registered_runner(tmp.path()).await;
14760 let winner = runner.state.candidates[0].clone();
14761 let cache = tmp.path().join("cache");
14762 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14763 runner.state.config.graph.timeout_resource_wait = 0;
14764 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14765 let crate::cache::AcquireOutcome::Acquired(held) =
14766 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14767 else {
14768 panic!("cache must be free")
14769 };
14770 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14771 from_run: "earlier".into(),
14772 generation: 1,
14773 mode: crate::recovery::Mode::VerifyRepair,
14774 reasons: Vec::new(),
14775 reviewed_head: None,
14776 rounds: Vec::new(),
14777 open_findings: Vec::new(),
14778 evidence: None,
14779 criteria_inherited: true,
14780 repair: crate::recovery::Repair::NotStarted,
14781 });
14782 runner.recovery_repair(&winner).await.unwrap();
14783 assert!(matches!(
14784 runner.state.recovery.as_ref().unwrap().repair,
14785 crate::recovery::Repair::WaitingForRestore { .. }
14786 ));
14787 drop(held);
14788 runner.state.resource_block = None;
14789 runner.state.config.graph.timeout_resource_wait = 1;
14790 runner.recovery_repair(&winner).await.unwrap();
14791 assert!(matches!(
14792 runner.state.recovery.as_ref().unwrap().repair,
14793 crate::recovery::Repair::Done { .. }
14794 ));
14795 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14796 }
14797
14798 #[tokio::test]
14799 #[cfg(unix)]
14800 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14801 use std::os::fd::AsRawFd as _;
14802 for graft in [false, true] {
14803 let tmp = tempfile::tempdir().unwrap();
14804 let mut runner = registered_runner(tmp.path()).await;
14805 runner.state.config.graph.timeout_resource_wait = 1;
14806 let winner = runner.state.candidates[0].clone();
14807 let marker = tmp.path().join("fixer-started");
14808 let mut fixer = spec("fixer");
14809 fixer.command = vec![
14810 "sh".into(),
14811 "-c".into(),
14812 format!("echo started >> '{}'; echo done", marker.display()),
14813 ];
14814 runner.roles.fixer = Some(vec![fixer]);
14815 if graft {
14816 let mut loser = winner.clone();
14817 loser.label = 'B';
14818 loser.index = 1;
14819 runner.state.candidates.push(loser);
14820 runner.state.config.graph.graft_runner_up = true;
14821 runner.state.tally = Some(Tally {
14822 first_choice: BTreeMap::new(),
14823 borda: BTreeMap::new(),
14824 winner: winner.label,
14825 rankings: 1,
14826 unanimous_initial: true,
14827 deliberated: false,
14828 changed_votes: 0,
14829 unanimous_final: true,
14830 tie_break: None,
14831 judges: 1,
14832 present: 1,
14833 quorum: 1,
14834 met_quorum: true,
14835 uncontested: None,
14836 });
14837 runner.state.judgements.push(crate::run::Judgement {
14838 judge: 1,
14839 seat: "judge-1".into(),
14840 agent: "judge".into(),
14841 ranking: vec![winner.label, 'B'],
14842 reasons: BTreeMap::new(),
14843 confidence: None,
14844 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14845 order: vec![0, 1],
14846 failed: None,
14847 duration_ms: 0,
14848 });
14849 } else {
14850 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14851 from_run: "earlier".into(),
14852 generation: 1,
14853 mode: crate::recovery::Mode::VerifyRepair,
14854 reasons: Vec::new(),
14855 reviewed_head: None,
14856 rounds: Vec::new(),
14857 open_findings: Vec::new(),
14858 evidence: None,
14859 criteria_inherited: true,
14860 repair: crate::recovery::Repair::NotStarted,
14861 });
14862 }
14863 let reader = crate::repo_lock::SeatLease::for_checkout(
14864 &winner.worktree,
14865 false,
14866 crate::config::test_coordination().as_ref(),
14867 )
14868 .await
14869 .unwrap();
14870 let pause = runner.pause.clone();
14871 let (result, ()) = tokio::join!(
14872 async {
14873 if graft {
14874 runner.graft().await
14875 } else {
14876 runner.recovery_repair(&winner).await
14877 }
14878 },
14879 async {
14880 tokio::time::sleep(Duration::from_millis(100)).await;
14881 pause.park();
14882 }
14883 );
14884 result.unwrap();
14885 assert!(!marker.exists(), "a blocked seat must not spawn");
14886 runner.state = RunState::load(&runner.state.id).unwrap();
14888 if graft {
14889 assert_eq!(
14890 runner.state.graft.as_ref().unwrap().outcome,
14891 GraftOutcome::WaitingForRestore
14892 );
14893 } else {
14894 assert!(matches!(
14895 runner.state.recovery.as_ref().unwrap().repair,
14896 crate::recovery::Repair::WaitingForRestore { .. }
14897 ));
14898 }
14899 drop(reader);
14900 runner.pause = Pause::new();
14901 runner.state.resource_block = None;
14902 let external =
14904 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14905 assert_eq!(
14906 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14907 0
14908 );
14909 let leftover = winner.worktree.join("leftover");
14910 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14911 let result = if graft {
14912 runner.graft().await
14913 } else {
14914 runner.recovery_repair(&winner).await
14915 };
14916 assert!(
14917 result
14918 .unwrap_err()
14919 .is::<crate::repo_lock::ResourceBlocked>()
14920 );
14921 assert!(leftover.exists());
14922 assert!(!marker.exists());
14923 drop(external);
14924 runner.state.resource_block = None;
14925 let result = if graft {
14926 runner.graft().await
14927 } else {
14928 runner.recovery_repair(&winner).await
14929 };
14930 result.unwrap();
14931 assert!(!leftover.exists(), "restore must precede retry");
14932 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14933 if graft {
14934 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14935 } else {
14936 assert!(matches!(
14937 runner.state.recovery.as_ref().unwrap().repair,
14938 crate::recovery::Repair::Done { .. }
14939 ));
14940 }
14941 }
14942 }
14943
14944 #[tokio::test]
14945 #[cfg(unix)]
14946 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14947 use std::os::fd::AsRawFd as _;
14948 let tmp = tempfile::tempdir().unwrap();
14949 let mut runner = registered_runner(tmp.path()).await;
14950 runner.state.config.graph.timeout_resource_wait = 1;
14951 let winner = runner.state.candidates[0].clone();
14952 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14953 runner.state.tally = Some(Tally {
14954 first_choice: BTreeMap::new(),
14955 borda: BTreeMap::new(),
14956 winner: winner.label,
14957 rankings: 0,
14958 unanimous_initial: false,
14959 deliberated: false,
14960 changed_votes: 0,
14961 unanimous_final: false,
14962 tie_break: None,
14963 judges: 0,
14964 present: 0,
14965 quorum: 0,
14966 met_quorum: true,
14967 uncontested: Some("fixture".into()),
14968 });
14969 runner.state.graft = Some(GraftRecord {
14970 pre: pre.clone(),
14971 post: None,
14972 lines: None,
14973 limit: None,
14974 agent: None,
14975 outcome: GraftOutcome::Running,
14976 requested: Vec::new(),
14977 adoption: None,
14978 adoption_error: None,
14979 });
14980 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14981 git::commit_all(&winner.worktree, "unfinished graft")
14982 .await
14983 .unwrap();
14984 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14985 let leftover = winner.worktree.join("leftover");
14986 std::fs::write(&leftover, "untrusted edits").unwrap();
14987 let path = runner.state.repo.join(".git/updater.lock");
14988 let external = std::fs::File::create(&path).unwrap();
14989 assert_eq!(
14990 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14991 0
14992 );
14993 assert!(
14994 runner
14995 .graft()
14996 .await
14997 .unwrap_err()
14998 .is::<crate::repo_lock::ResourceBlocked>()
14999 );
15000 assert!(matches!(
15001 runner.state.graft.as_ref().unwrap().outcome,
15002 GraftOutcome::Running
15003 ));
15004 assert_eq!(
15005 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15006 unfinished
15007 );
15008 assert!(leftover.exists());
15009 drop(external);
15010 std::fs::remove_file(&path).unwrap();
15011 std::fs::create_dir(&path).unwrap();
15012 assert!(runner.graft().await.is_err());
15013 assert!(matches!(
15014 runner.state.graft.as_ref().unwrap().outcome,
15015 GraftOutcome::Running
15016 ));
15017 assert_eq!(
15018 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15019 unfinished
15020 );
15021 assert!(leftover.exists());
15022 std::fs::remove_dir(&path).unwrap();
15023 runner.graft().await.unwrap();
15024 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15025 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15026 assert!(!leftover.exists());
15027 }
15028
15029 #[tokio::test]
15030 #[cfg(unix)]
15031 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15032 use std::os::fd::AsRawFd as _;
15033 let tmp = tempfile::tempdir().unwrap();
15034 let mut runner = registered_runner(tmp.path()).await;
15035 runner.state.prep_complete = Some(false);
15036 runner.state.config.graph.timeout_resource_wait = 1;
15037 runner.state.save().unwrap();
15038 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15039 assert_eq!(
15040 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15041 0
15042 );
15043 runner.execute().await.unwrap();
15044 let disk = RunState::load(&runner.state.id).unwrap();
15045 assert!(disk.driver_exited);
15046 assert_eq!(disk.driver_pid, None);
15047 assert_eq!(runner.state.status, RunStatus::Blocked);
15048 assert_eq!(
15049 runner.state.resource_block.as_ref().unwrap().kind,
15050 "repository coordination"
15051 );
15052 assert!(runner.state.prep_failure.is_none());
15053 assert!(runner.state.seats.is_empty());
15054 runner.pause.park();
15055 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15058 .await
15059 .unwrap()
15060 .unwrap();
15061 assert!(runner.state.parked);
15062 assert!(runner.state.prep_failure.is_none());
15063 let disk = RunState::load(&runner.state.id).unwrap();
15064 assert!(disk.driver_exited);
15065 assert_eq!(disk.driver_pid, None);
15066 }
15067
15068 #[tokio::test]
15069 #[cfg(unix)]
15070 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15071 use std::os::fd::AsRawFd as _;
15072 let tmp = tempfile::tempdir().unwrap();
15073 let mut runner = registered_runner(tmp.path()).await;
15074 runner.state.prep_complete = Some(false);
15075 runner.state.config.graph.timeout_resource_wait = 30;
15076 runner.state.save().unwrap();
15077 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15078 assert_eq!(
15079 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15080 0
15081 );
15082 assert!(
15083 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15084 .await
15085 .is_err()
15086 );
15087 let disk = RunState::load(&runner.state.id).unwrap();
15088 assert!(disk.driver_exited);
15089 assert_eq!(disk.driver_pid, None);
15090 assert!(disk.status.resumable());
15091 assert!(runner.state.driver_exited);
15092 assert_eq!(runner.state.driver_pid, None);
15093 runner.state.save().unwrap();
15094 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15095 }
15096
15097 #[tokio::test]
15098 #[cfg(unix)]
15099 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15100 let tmp = tempfile::tempdir().unwrap();
15101 let mut runner = registered_runner(tmp.path()).await;
15102 let winner = runner.state.candidates[0].clone();
15103 let cache = tmp.path().join("cache");
15104 let started = tmp.path().join("pre-gate-started");
15105 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15106 runner.state.config.verify.gate =
15107 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15108 let home = crate::run::home();
15109 let owner =
15110 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15111 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15112 crate::cache::AcquireOutcome::Acquired(g) => g,
15113 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15114 };
15115 let mut gate = Box::pin(runner.run_gate(&winner));
15116 assert!(
15117 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15118 .await
15119 .is_err()
15120 );
15121 assert!(!started.exists(), "pre-gate must wait for the cache too");
15122 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15126 let editor = crate::repo_lock::SeatLease::checkout_with(
15127 &winner.worktree,
15128 true,
15129 crate::config::test_coordination().as_ref(),
15130 &wait,
15131 )
15132 .await
15133 .unwrap();
15134 drop(editor);
15135 guard.release();
15136 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15137 .await
15138 .unwrap()
15139 .unwrap();
15140 drop(gate);
15141 assert!(outcomes.iter().all(CommandOutcome::ok));
15142 assert!(started.exists());
15143 }
15144
15145 #[tokio::test]
15146 #[cfg(unix)]
15147 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15148 use std::os::fd::AsRawFd as _;
15149
15150 ask_test_home();
15151 let tmp = tempfile::tempdir().unwrap();
15152 let repo = tmp.path().join("repo");
15153 std::fs::create_dir_all(&repo).unwrap();
15154 init_repo(&repo);
15155 let common = repo.join(".git");
15156 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15157 let mut runner = runner_at(RunStatus::Prep);
15158 runner.state.repo = repo.clone();
15159 runner.state.config.coordination = crate::config::test_coordination();
15160 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15161 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15162 runner.state.config.graph.advise = false;
15163 runner.roles.implementers = vec![spec("implement")];
15164 runner.roles.judges = vec![spec("judge")];
15165 runner.prep().await.unwrap();
15166 let winner = runner.state.candidates[0].clone();
15167 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15170 let completed = tmp.path().join("seat-completed");
15171 runner.roles.implementers[0].command = vec![
15172 "sh".into(),
15173 "-c".into(),
15174 format!("touch '{}'; echo completed", completed.display()),
15175 ];
15176 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15177 assert_eq!(
15178 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15179 0
15180 );
15181 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15182 let mut implement = Box::pin(runner.implement());
15183 tokio::select! {
15184 result = &mut implement => panic!("rescue must wait: {result:?}"),
15185 result = tokio::time::timeout(Duration::from_secs(10), async {
15186 while !completed.exists() {
15187 tokio::time::sleep(Duration::from_millis(20)).await;
15188 }
15189 }) => result.unwrap(),
15190 }
15191 assert!(
15192 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15193 .await
15194 .is_err()
15195 );
15196 assert_eq!(
15197 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15198 before
15199 );
15200 assert_eq!(
15201 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15202 0
15203 );
15204 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15205 .await
15206 .unwrap()
15207 .unwrap();
15208 drop(implement);
15209 assert_ne!(
15210 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15211 before
15212 );
15213 assert!(git::is_clean(&winner.worktree).await.unwrap());
15214
15215 let started = tmp.path().join("pre-gate-started");
15216 let proceed = tmp.path().join("pre-gate-proceed");
15217 let checked = tmp.path().join("gate-ran");
15218 runner.state.config.verify.pre_gate = vec![format!(
15219 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15220 started.display(),
15221 proceed.display()
15222 )];
15223 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15224 assert_eq!(
15225 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15226 0
15227 );
15228 let mut gate = Box::pin(runner.run_gate(&winner));
15229 assert!(
15230 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15231 .await
15232 .is_err()
15233 );
15234 assert!(!started.exists());
15235 assert!(!checked.exists());
15236 assert!(git::is_clean(&winner.worktree).await.unwrap());
15237 assert_eq!(
15238 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15239 0
15240 );
15241 tokio::select! {
15242 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15243 result = tokio::time::timeout(Duration::from_secs(10), async {
15244 while !started.exists() {
15245 tokio::time::sleep(Duration::from_millis(20)).await;
15246 }
15247 }) => result.unwrap(),
15248 }
15249 assert_eq!(
15250 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15251 -1,
15252 "mechanical edits must hold the real lock"
15253 );
15254 std::fs::write(proceed, "ready").unwrap();
15255 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15256 .await
15257 .unwrap()
15258 .unwrap();
15259 drop(gate);
15260 assert!(outcomes.iter().all(CommandOutcome::ok));
15261 assert!(checked.exists());
15262 assert!(runner.state.pre_gate_commit.is_some());
15263 assert!(git::is_clean(&winner.worktree).await.unwrap());
15264 assert_eq!(
15265 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15266 0,
15267 "gate completion must release coordination"
15268 );
15269 drop(outsider);
15270 std::fs::remove_file(common.join("updater.lock")).unwrap();
15271 std::fs::create_dir(common.join("updater.lock")).unwrap();
15272 std::fs::remove_file(&started).unwrap();
15273 std::fs::remove_file(&checked).unwrap();
15274 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15275 let error = runner.run_gate(&winner).await.unwrap_err();
15276 assert!(
15277 error
15278 .to_string()
15279 .contains("acquire mandatory repository lock")
15280 );
15281 assert!(
15282 !started.exists(),
15283 "failed acquisition must prevent pre-gate"
15284 );
15285 assert!(!checked.exists(), "failed acquisition must prevent gate");
15286 assert!(git::is_clean(&winner.worktree).await.unwrap());
15287 assert_eq!(
15288 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15289 head
15290 );
15291 }
15292
15293 #[tokio::test]
15294 #[cfg(unix)]
15295 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15296 use std::os::fd::AsRawFd as _;
15297 use std::time::Duration;
15298
15299 ask_test_home();
15300 let tmp = tempfile::tempdir().unwrap();
15301 let repo = tmp.path().join("repo");
15302 std::fs::create_dir_all(&repo).unwrap();
15303 init_repo(&repo);
15304 let common = repo.join(".git");
15305 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15306 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15307 assert_eq!(
15308 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15309 0
15310 );
15311 let mut runner = runner_at(RunStatus::Prep);
15312 runner.state.repo = repo.clone();
15313 runner.state.config.coordination = crate::config::test_coordination();
15314 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15315 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15316 runner.state.config.graph.advise = true;
15317 runner.state.config.graph.advisors = 1;
15318 runner.roles.implementers = vec![spec("implement")];
15319 runner.roles.judges = vec![spec("judge")];
15320 let root = runner.state.worktree_root();
15321 let started = tmp.path().join("setup-started");
15322 let proceed = tmp.path().join("setup-proceed");
15323 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15324 run: Some(format!(
15325 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15326 started.display(),
15327 proceed.display()
15328 )),
15329 ..Default::default()
15330 }];
15331 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15332 let config = git::git(&repo, &["config", "--local", "--list"])
15333 .await
15334 .unwrap();
15335 let mut prep = Box::pin(runner.prep());
15336 assert!(
15337 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15338 .await
15339 .is_err()
15340 );
15341 assert!(
15342 !root.exists(),
15343 "contention must prevent even checkout creation"
15344 );
15345 assert!(!started.exists(), "contention must prevent setup");
15346 assert_eq!(
15347 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15348 branches
15349 );
15350 assert_eq!(
15351 git::git(&repo, &["config", "--local", "--list"])
15352 .await
15353 .unwrap(),
15354 config
15355 );
15356 drop(lock);
15357 tokio::select! {
15358 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15359 result = tokio::time::timeout(Duration::from_secs(10), async {
15360 while !started.exists() {
15361 tokio::time::sleep(Duration::from_millis(20)).await;
15362 }
15363 }) => result.unwrap(),
15364 }
15365 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15366 assert_eq!(
15367 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15368 -1
15369 );
15370 assert!(
15371 tokio::time::timeout(
15372 Duration::from_millis(100),
15373 crate::repo_lock::SeatLease::for_checkout(
15374 &repo,
15375 true,
15376 crate::config::test_coordination().as_ref()
15377 )
15378 )
15379 .await
15380 .is_err(),
15381 "preparation must also exclude local editors"
15382 );
15383 std::fs::write(proceed, "ready").unwrap();
15384 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15385 .await
15386 .unwrap()
15387 .unwrap();
15388 drop(prep);
15389 assert_eq!(runner.state.prep_complete, Some(true));
15390 assert!(root.join("judge-1").exists());
15391 assert!(root.join("advisor-1").exists());
15392 assert_eq!(
15393 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15394 0
15395 );
15396 }
15397
15398 #[tokio::test]
15399 #[cfg(unix)]
15400 async fn preparation_lock_failure_prevents_repository_changes() {
15401 ask_test_home();
15402 let tmp = tempfile::tempdir().unwrap();
15403 let repo = tmp.path().join("repo");
15404 std::fs::create_dir_all(&repo).unwrap();
15405 init_repo(&repo);
15406 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15407 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15410 let mut runner = runner_at(RunStatus::Prep);
15411 runner.state.repo = repo.clone();
15412 runner.state.config.coordination = crate::config::test_coordination();
15413 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15414 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15415 runner.roles.implementers = vec![spec("implement")];
15416 let error = runner.prep().await.unwrap_err();
15417 assert!(
15418 error
15419 .to_string()
15420 .contains("acquire mandatory repository lock")
15421 );
15422 assert!(!runner.state.worktree_root().exists());
15423 assert!(runner.state.candidates.is_empty());
15424 assert!(!repo.join(".git/config.worktree").exists());
15425 assert_eq!(
15426 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15427 "* main"
15428 );
15429 }
15430
15431 #[tokio::test]
15432 async fn preparation_records_creation_intent_before_git_can_fail() {
15433 ask_test_home();
15434 let tmp = tempfile::tempdir().unwrap();
15435 let repo = tmp.path().join("repo");
15436 std::fs::create_dir_all(&repo).unwrap();
15437 init_repo(&repo);
15438 let mut runner = runner_at(RunStatus::Prep);
15439 runner.state.repo = repo.clone();
15440 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15441 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15442 runner.state.config.blind.commit_msg_hook = false;
15443 runner.roles.implementers = vec![spec("one")];
15444 let label = blind::assign_labels(1, runner.state.seed)[0];
15445 let branch = runner.state.branch_for(label);
15446 git::git(&repo, &["branch", &branch]).await.unwrap();
15449 assert!(runner.prep().await.is_err());
15450 let saved = RunState::load(&runner.state.id).unwrap();
15451 assert!(saved.candidates.is_empty());
15452 let event = saved
15453 .events
15454 .iter()
15455 .find(|e| e.node == "prep-intent")
15456 .unwrap();
15457 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15458 assert_eq!(intent["branch"], branch);
15459 assert_eq!(intent["start"], runner.state.base_commit);
15460 assert_eq!(
15461 intent["worktree"],
15462 runner
15463 .state
15464 .worktree_root()
15465 .join(format!("cand-{label}"))
15466 .to_string_lossy()
15467 .as_ref()
15468 );
15469 assert_eq!(intent["picks"], serde_json::json!([]));
15470 }
15471
15472 #[tokio::test]
15473 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15474 ask_test_home();
15475 let tmp = tempfile::tempdir().unwrap();
15476 let repo = tmp.path().join("repo");
15477 std::fs::create_dir_all(&repo).unwrap();
15478 init_repo(&repo);
15479 let mut runner = runner_at(RunStatus::Prep);
15480 runner.state.repo = repo.clone();
15481 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15482 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15483 runner.state.sources.pick_commits.push(refs::Input {
15484 token: "missing".into(),
15485 sha: "0".repeat(40),
15486 provenance: refs::Provenance::ExplicitCli,
15487 });
15488 assert!(runner.prep().await.is_err());
15489 assert!(runner.state.candidates.is_empty());
15490 assert!(!runner.state.worktree_root().exists());
15491 }
15492
15493 #[tokio::test]
15494 async fn prose_and_legacy_inference_never_import_into_candidates() {
15495 ask_test_home();
15496 let tmp = tempfile::tempdir().unwrap();
15497 let repo = tmp.path().join("repo");
15498 std::fs::create_dir_all(&repo).unwrap();
15499 init_repo(&repo);
15500 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15501 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15502 .await
15503 .unwrap();
15504 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15505 git::git(&repo, &["add", "."]).await.unwrap();
15506 git::git(&repo, &["commit", "-m", "historical implementation"])
15507 .await
15508 .unwrap();
15509 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15510 git::git(&repo, &["checkout", "main"]).await.unwrap();
15511 for instruction in [
15512 format!("Example: magi/history/A at {source}"),
15513 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15514 format!("Never import magi/history/A or {source}"),
15515 format!("Conversation answer: I mentioned {source} only as background"),
15516 ] {
15517 let mut runner = runner_at(RunStatus::Prep);
15518 runner.state.repo = repo.clone();
15519 runner.state.base_commit = base.clone();
15520 runner.state.instruction = instruction.clone();
15521 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15522 runner.state.config.blind.commit_msg_hook = false;
15523 runner.state.seeds = vec![refs::Seed {
15524 token: "magi/history/A".into(),
15525 sha: source.clone(),
15526 kind: refs::SeedKind::Unmerged,
15527 branch: true,
15528 detail: String::new(),
15529 }];
15530 runner.roles.implementers = vec![spec("one")];
15531 assert_eq!(seeded_instruction(&runner.state), instruction);
15532 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15534 legacy.as_object_mut().unwrap().remove("sources");
15535 runner.state = serde_json::from_value(legacy).unwrap();
15536 assert_eq!(runner.state.sources, refs::Sources::default());
15537 runner.state.save().unwrap();
15538 runner.state = RunState::load(&runner.state.id).unwrap();
15539 runner.prep().await.unwrap();
15540 for candidate in &runner.state.candidates {
15541 assert_eq!(
15542 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15543 base
15544 );
15545 assert!(!candidate.worktree.join("unwanted").exists());
15546 }
15547 }
15548 }
15549
15550 #[tokio::test]
15551 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15552 ask_test_home();
15553 let tmp = tempfile::tempdir().unwrap();
15554 let repo = tmp.path().join("repo");
15555 std::fs::create_dir_all(&repo).unwrap();
15556 init_repo(&repo);
15557 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15558 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15559 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15560 git::git(&repo, &["add", "."]).await.unwrap();
15561 git::git(&repo, &["commit", "-m", "required seed"])
15562 .await
15563 .unwrap();
15564 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15565 git::git(&repo, &["checkout", "main"]).await.unwrap();
15566 let mut runner = runner_at(RunStatus::Prep);
15567 runner.state.repo = repo.clone();
15568 runner.state.base_commit = base.clone();
15569 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15570 runner.state.config.graph.advise = false;
15571 runner.state.config.blind.commit_msg_hook = false;
15572 runner.state.sources =
15573 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15574 .await
15575 .unwrap();
15576 runner.roles.implementers = vec![spec("one"), spec("two")];
15577 runner.roles.judges = vec![spec("judge")];
15578 let label = blind::assign_labels(2, runner.state.seed)[0];
15579 let branch = runner.state.branch_for(label);
15580 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15581 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15582 .await
15583 .unwrap();
15584 runner.state.candidates.push(Candidate {
15585 index: 0,
15586 label,
15587 agent: "one".into(),
15588 branch,
15589 worktree,
15590 summary: String::new(),
15591 stat: String::new(),
15592 files: 0,
15593 commits: 0,
15594 empty: false,
15595 failed: None,
15596 verified_noop: None,
15597 duration_ms: 0,
15598 folded: false,
15599 });
15600 runner.state.save().unwrap();
15601 runner.state = RunState::load(&runner.state.id).unwrap();
15603 assert_eq!(runner.state.prep_complete, Some(false));
15604 runner.prep().await.unwrap();
15605 assert_eq!(runner.state.prep_complete, Some(true));
15606 assert_eq!(runner.state.candidates.len(), 2);
15607 for candidate in &runner.state.candidates {
15608 assert_eq!(
15609 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15610 git::tree_of(&repo, &seed).await.unwrap()
15611 );
15612 assert_eq!(
15613 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15614 "required commit"
15615 );
15616 }
15617 assert!(runner.state.worktree_root().join("judge-1").exists());
15618 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15619 runner.prep().await.unwrap();
15620 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15621
15622 runner.state.prep_complete = Some(false);
15623 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15624 run: Some(format!(
15625 "test -f '{}'",
15626 tmp.path().join("dependency-ready").display()
15627 )),
15628 ..Default::default()
15629 }];
15630 assert!(runner.prep().await.is_err());
15631 assert!(runner.state.prep_failure.is_some());
15632 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15633 runner.state = RunState::load(&runner.state.id).unwrap();
15634 assert!(
15635 runner.prep().await.is_err(),
15636 "correcting the cause alone must not auto-retry"
15637 );
15638 let failed_at = runner.state.events.last().unwrap().at;
15639 assert!(
15640 runner.retry_preparation(failed_at).is_err(),
15641 "old approvals cannot unlock the hold"
15642 );
15643 let mut task = crate::queue::Task::new(
15644 "retry".into(),
15645 "retry setup".into(),
15646 repo.clone(),
15647 crate::queue::Source::Human,
15648 );
15649 task.start(runner.state.id.clone());
15650 task.hold_machine(Some("setup failed".into()));
15651 task.release_operator("cli", None);
15652 runner
15653 .retry_preparation(task.resumption.as_ref().unwrap().at)
15654 .unwrap();
15655 runner.prep().await.unwrap();
15656 assert!(runner.state.prep_failure.is_none());
15657 assert_eq!(runner.state.prep_complete, Some(true));
15658 assert_eq!(runner.state.candidates.len(), 2);
15659 assert!(runner.state.worktree_root().join("judge-1").exists());
15660 for candidate in &runner.state.candidates {
15661 assert_eq!(
15662 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15663 git::tree_of(&repo, &seed).await.unwrap()
15664 );
15665 }
15666 }
15667
15668 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15669 let mut heads = Vec::new();
15670 for candidate in candidates {
15671 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15672 }
15673 heads
15674 }
15675
15676 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15677 ask_test_home();
15678 let tmp = tempfile::tempdir().unwrap();
15679 let repo = tmp.path().join("repo");
15680 std::fs::create_dir(&repo).unwrap();
15681 init_repo(&repo);
15682 let mut runner = runner_at(RunStatus::Reviewing);
15683 runner.state.repo = repo.clone();
15684 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15685 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15686 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15687 let branch = format!("candidate-{label}");
15688 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15689 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15690 .await
15691 .unwrap();
15692 if label == 'B' {
15693 std::fs::write(worktree.join("change"), "winner").unwrap();
15694 git::rescue_commit(&worktree, "candidate change")
15695 .await
15696 .unwrap();
15697 }
15698 runner.state.candidates.push(Candidate {
15699 index,
15700 label,
15701 agent: "alpha".into(),
15702 branch,
15703 worktree,
15704 summary: "completed reply".into(),
15705 stat: String::new(),
15706 files: usize::from(label == 'B'),
15707 commits: usize::from(label == 'B'),
15708 empty: label == 'A',
15709 failed: None,
15710 verified_noop: None,
15711 duration_ms: 1,
15712 folded: false,
15713 });
15714 }
15715 runner.state.judge_skipped = true;
15716 runner.state.tally = Some(Tally {
15717 first_choice: BTreeMap::new(),
15718 borda: BTreeMap::new(),
15719 winner: 'B',
15720 rankings: 0,
15721 unanimous_initial: false,
15722 deliberated: false,
15723 changed_votes: 0,
15724 unanimous_final: false,
15725 tie_break: None,
15726 judges: 0,
15727 present: 0,
15728 quorum: 0,
15729 met_quorum: true,
15730 uncontested: Some("only B produced a change".into()),
15731 });
15732 runner.state.save().unwrap();
15733 (tmp, runner)
15734 }
15735
15736 #[tokio::test]
15737 #[cfg(unix)]
15738 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15739 for (solo, failed_checkpoint) in
15740 [(true, false), (false, false), (true, true), (false, true)]
15741 {
15742 let (_tmp, mut runner) = preservation_resume_fixture().await;
15743 if solo {
15744 runner.state.candidates.retain(|c| c.label == 'A');
15745 }
15746 runner.state.candidates[0].empty = false;
15747 runner.state.candidates[0].summary.clear();
15748 runner.state.tally = None;
15749 runner.state.judge_skipped = false;
15750 runner.state.status = RunStatus::Implementing;
15751 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15752 let mut implementer = spec("alpha");
15753 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15754 runner.roles.implementers = vec![implementer];
15755 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15756 if failed_checkpoint {
15757 std::fs::create_dir_all(&patch).unwrap();
15758 }
15759 runner.state.save().unwrap();
15760 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15761 drop(pressure);
15762 runner.state = RunState::load(&runner.state.id).unwrap();
15763 assert_eq!(runner.state.status, RunStatus::Blocked);
15764 assert!(runner.state.interrupted_implementations.contains(&'A'));
15765 assert_eq!(runner.state.candidates[0].commits, 0);
15766 assert!(!runner.state.candidates[0].empty);
15767 assert!(runner.state.candidates[0].failed.is_none());
15768 if failed_checkpoint {
15769 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15770 std::fs::remove_dir(patch).unwrap();
15771 }
15772 runner.state.resource_block = None;
15773 runner.state.config.disk.min_free_bytes = 0;
15774 runner.roles.implementers[0].command = vec![
15775 "sh".into(),
15776 "-c".into(),
15777 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15778 ];
15779 runner.implement().await.unwrap();
15780 assert!(runner.state.interrupted_implementations.is_empty());
15781 assert!(runner.state.candidate_preservation.is_empty());
15782 assert_eq!(runner.state.candidates[0].commits, 1);
15783 assert!(!runner.state.candidates[0].empty);
15784 assert!(
15785 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15786 .unwrap()
15787 .contains("retried empty interrupted seat")
15788 );
15789 if !solo {
15790 assert_eq!(runner.state.candidates[1].commits, 1);
15791 }
15792 }
15793 }
15794
15795 #[tokio::test]
15796 #[cfg(unix)]
15797 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15798 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15799 let (_tmp, mut runner) = preservation_resume_fixture().await;
15800 runner.state.candidates.retain(|c| c.label == 'A');
15801 runner.state.candidates[0].empty = false;
15802 runner.state.candidates[0].summary.clear();
15803 runner.state.tally = None;
15804 runner.state.judge_skipped = false;
15805 runner.state.status = RunStatus::Implementing;
15806 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15809 let wt = runner.state.candidates[0].worktree.clone();
15810 std::fs::write(
15811 wt.join("interrupted-source"),
15812 "retain implementation work\n",
15813 )
15814 .unwrap();
15815 if committed {
15816 git::rescue_commit(&wt, "implementation checkpoint")
15817 .await
15818 .unwrap();
15819 }
15820 let lock = git::git(
15821 &wt,
15822 &[
15823 "rev-parse",
15824 "--path-format=absolute",
15825 "--git-path",
15826 "index.lock",
15827 ],
15828 )
15829 .await
15830 .unwrap();
15831 if block_rescue {
15832 std::fs::write(&lock, "fixture-owned lock").unwrap();
15833 }
15834 let mut implementer = spec("alpha");
15835 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15836 runner.roles.implementers = vec![implementer];
15837 runner.state.save().unwrap();
15838 let result = runner.implement().await;
15839 drop(pressure);
15840 assert_eq!(result.is_err(), block_rescue);
15841 assert_eq!(
15842 runner.state.resource_block.as_ref().unwrap().kind,
15843 "disk pressure"
15844 );
15845 assert!(
15846 runner
15847 .state
15848 .events
15849 .iter()
15850 .any(|e| e.message.contains("agent execution interrupted"))
15851 );
15852 assert!(
15853 runner
15854 .state
15855 .dir()
15856 .join("artifacts/candidate-A-recovery/staged.patch")
15857 .exists()
15858 );
15859 if block_rescue {
15860 let saved = RunState::load(&runner.state.id).unwrap();
15861 assert!(saved.candidate_preservation.contains_key(&'A'));
15862 assert!(!saved.candidates[0].empty);
15863 assert_eq!(
15864 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15865 "retain implementation work\n"
15866 );
15867 std::fs::remove_file(lock).unwrap();
15868 runner.state = saved;
15869 } else {
15870 assert_eq!(runner.state.status, RunStatus::Blocked);
15871 assert_eq!(runner.state.candidates[0].commits, 1);
15872 }
15873 runner.roles.implementers.clear();
15876 runner.state.config.disk.min_free_bytes = 0;
15877 runner.state.resource_block = None;
15878 runner.implement().await.unwrap();
15879 assert!(runner.state.candidate_preservation.is_empty());
15880 assert_eq!(runner.state.candidates[0].commits, 1);
15881 assert!(!runner.state.candidates[0].empty);
15882 assert!(git::is_clean(&wt).await.unwrap());
15883 assert!(
15884 crate::run::read_artifact(&runner.state, "cand-A.patch")
15885 .unwrap()
15886 .contains("retain implementation work")
15887 );
15888 }
15889 }
15890
15891 #[tokio::test]
15892 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15893 let (_tmp, mut runner) = preservation_resume_fixture().await;
15894 runner.implement().await.unwrap();
15896 assert!(runner.state.candidate_preservation.is_empty());
15897 assert!(
15898 !runner
15899 .state
15900 .dir()
15901 .join("artifacts/candidate-A-recovery")
15902 .exists()
15903 );
15904 runner.fold_losers().await.unwrap();
15905 assert!(!runner.state.candidates[0].worktree.exists());
15906 for schema in [14, crate::run::SCHEMA] {
15907 runner.state.schema = schema;
15908 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15909 runner.state.save().unwrap();
15910 runner.state = RunState::load(&runner.state.id).unwrap();
15911 runner.implement().await.unwrap();
15912 assert!(runner.state.candidate_preservation.is_empty());
15913 assert!(runner.state.candidates[0].folded);
15914 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15915 }
15916 }
15917
15918 #[tokio::test]
15919 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15920 let (_tmp, mut runner) = preservation_resume_fixture().await;
15921 runner.state.candidates.retain(|c| c.label == 'A');
15922 runner.state.tally = None;
15923 runner.state.judge_skipped = false;
15924 let candidate = &mut runner.state.candidates[0];
15925 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15926 candidate.failed = Some("deployment validation unavailable".into());
15927 std::fs::write(
15928 candidate.worktree.join("repair"),
15929 "future launcher policy\n",
15930 )
15931 .unwrap();
15932 runner
15933 .state
15934 .candidate_preservation
15935 .insert('A', "awaiting persistence".into());
15936 runner.persist_candidates().await.unwrap();
15937 let candidate = &runner.state.candidates[0];
15938 assert!(!candidate.empty);
15939 assert_eq!(candidate.commits, 1);
15940 assert_eq!(candidate.files, 1);
15941 assert!(candidate.failed.is_none());
15942 assert!(candidate.summary.contains("EPERM"));
15943 assert!(
15944 crate::run::read_artifact(&runner.state, "cand-A.patch")
15945 .unwrap()
15946 .contains("future launcher policy")
15947 );
15948 assert!(runner.state.candidate_preservation.is_empty());
15949 }
15950
15951 #[tokio::test]
15952 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15953 let (_tmp, mut runner) = preservation_resume_fixture().await;
15954 let wt = runner.state.candidates[0].worktree.clone();
15955 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15956 runner.state.schema = 14;
15957 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15958 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15959 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15960 for _ in 0..2 {
15961 assert!(runner.implement().await.is_err());
15962 assert_eq!(runner.state.status, RunStatus::Blocked);
15963 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15964 assert!(
15965 runner.state.candidates[0].empty,
15966 "old verdict must remain unusable"
15967 );
15968 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15969 assert!(runner.state.judge_skipped);
15970 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15971 assert!(
15972 crate::run::read_artifact(&runner.state, "cand-A.patch")
15973 .unwrap()
15974 .contains("owned edits")
15975 );
15976 assert!(wt.exists());
15977 assert_eq!(
15978 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15979 .await
15980 .unwrap(),
15981 1
15982 );
15983 runner.state = RunState::load(&runner.state.id).unwrap();
15984 }
15985 }
15986
15987 #[tokio::test]
15988 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15989 let (_tmp, mut runner) = preservation_resume_fixture().await;
15990 let wt = runner.state.candidates[0].worktree.clone();
15991 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15992 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15993 let index_lock = PathBuf::from(
15994 git::git(
15995 &wt,
15996 &[
15997 "rev-parse",
15998 "--path-format=absolute",
15999 "--git-path",
16000 "index.lock",
16001 ],
16002 )
16003 .await
16004 .unwrap(),
16005 );
16006 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16007 runner.state.schema = 14;
16008 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16009 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16010 assert!(runner.implement().await.is_err());
16011 assert!(runner.after_implement().is_err());
16012 assert!(runner.fold_losers().await.is_err());
16013 assert_eq!(
16014 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16015 "failed seat edits"
16016 );
16017 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16018 runner.state = RunState::load(&runner.state.id).unwrap();
16019 std::fs::remove_file(index_lock).unwrap();
16020 assert!(runner.implement().await.is_err());
16022 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16023 assert!(runner.state.candidates[0].empty);
16024 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16025 assert!(
16026 crate::run::read_artifact(&runner.state, "cand-A.patch")
16027 .unwrap()
16028 .contains("failed seat edits")
16029 );
16030 assert!(wt.exists());
16031 }
16032
16033 #[tokio::test]
16034 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16035 let (_tmp, mut runner) = preservation_resume_fixture().await;
16036 let repo = runner.state.repo.clone();
16037 let common = git::git(
16038 &repo,
16039 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16040 )
16041 .await
16042 .unwrap();
16043 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16044 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16045 assert!(runner.fold_losers().await.is_err());
16046 assert!(!runner.state.candidates[0].worktree.exists());
16047 assert!(!runner.state.candidates[0].folded);
16048 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16049 runner.state = RunState::load(&runner.state.id).unwrap();
16050 std::fs::remove_file(branch_lock).unwrap();
16051 let judge = runner.state.worktree_root().join("judge-1");
16052 git::worktree_add_detached(&repo, &judge, "main")
16053 .await
16054 .unwrap();
16055 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16056 runner.roles.judges = vec![spec("judge")];
16057 assert!(runner.fold_losers().await.is_err());
16058 runner.state = RunState::load(&runner.state.id).unwrap();
16059 assert!(
16060 runner.state.candidates[0].folded,
16061 "candidate progress saved before judge failure"
16062 );
16063 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16064 assert!(judge.join("uncommitted").exists());
16065 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16066 runner.fold_losers().await.unwrap();
16067 assert!(!judge.exists());
16068 runner.fold_losers().await.unwrap();
16069 }
16070
16071 #[tokio::test]
16072 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16073 let (_tmp, mut runner) = preservation_resume_fixture().await;
16074 runner.state.candidates.retain(|c| c.label == 'B');
16075 runner.state.tally = None;
16076 runner.state.judge_skipped = false;
16077 runner.state.prep_complete = Some(true);
16078 runner.state.config.graph.advise = false;
16079 runner.state.config.graph.selected_implementer = Some("alpha".into());
16080 let wt = runner.state.candidates[0].worktree.clone();
16081 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16082 runner
16083 .state
16084 .candidate_preservation
16085 .insert('B', "awaiting persistence".into());
16086 let index_lock = git::git(
16087 &wt,
16088 &[
16089 "rev-parse",
16090 "--path-format=absolute",
16091 "--git-path",
16092 "index.lock",
16093 ],
16094 )
16095 .await
16096 .unwrap();
16097 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16098 runner.state.save().unwrap();
16099 assert!(runner.execute_graph().await.is_err());
16100 assert_eq!(runner.state.status, RunStatus::Blocked);
16101 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16102 std::fs::remove_file(index_lock).unwrap();
16103 for _ in 0..2 {
16106 runner.state = RunState::load(&runner.state.id).unwrap();
16107 runner.pause.park();
16108 runner.execute_graph().await.unwrap();
16109 assert_eq!(runner.state.status, RunStatus::Implementing);
16110 assert!(runner.state.candidate_preservation.is_empty());
16111 assert_eq!(runner.state.candidates[0].commits, 2);
16112 assert!(runner.state.tally.is_none());
16113 }
16114 runner.implement().await.unwrap();
16115 assert_eq!(runner.state.status, RunStatus::Judging);
16116 runner.state.status = RunStatus::Blocked;
16118 runner.state.candidates[0].failed = Some("implementation failed".into());
16119 runner.state.save().unwrap();
16120 let error = runner.execute_graph().await.unwrap_err();
16121 assert!(
16122 error
16123 .to_string()
16124 .contains("selected implementation is held")
16125 );
16126 }
16127
16128 #[tokio::test]
16129 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16130 let (_tmp, mut runner) = preservation_resume_fixture().await;
16131 let repo = runner.state.repo.clone();
16132 let candidate = runner.state.candidates[0].clone();
16133 let registered = git::worktree_holding(&repo, &candidate.branch)
16136 .await
16137 .unwrap()
16138 .unwrap();
16139 assert_eq!(
16140 registered.canonicalize().unwrap(),
16141 candidate.worktree.canonicalize().unwrap()
16142 );
16143 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16144 assert_eq!(
16145 git::worktree_holding(&repo, &candidate.branch)
16146 .await
16147 .unwrap(),
16148 Some(registered)
16149 );
16150 assert!(
16151 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16152 "the stale registration initially keeps the branch checked out"
16153 );
16154 runner.fold_losers().await.unwrap();
16155 assert!(runner.state.candidates[0].folded);
16156 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16157 assert_eq!(
16158 git::worktree_holding(&repo, &candidate.branch)
16159 .await
16160 .unwrap(),
16161 None
16162 );
16163 runner.state = RunState::load(&runner.state.id).unwrap();
16164 runner.fold_losers().await.unwrap();
16165 }
16166
16167 #[tokio::test]
16168 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16169 let (_tmp, mut runner) = preservation_resume_fixture().await;
16170 let repo = runner.state.repo.clone();
16171 let candidate = runner.state.candidates[0].clone();
16172 let winner = runner.state.candidates[1].clone();
16173 let registered = git::worktree_holding(&repo, &candidate.branch)
16174 .await
16175 .unwrap()
16176 .unwrap();
16177 assert_eq!(
16178 registered.canonicalize().unwrap(),
16179 candidate.worktree.canonicalize().unwrap()
16180 );
16181 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16182 assert_eq!(
16183 git::worktree_holding(&repo, &candidate.branch)
16184 .await
16185 .unwrap(),
16186 Some(registered)
16187 );
16188 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16189
16190 for _ in 0..2 {
16191 fold_run(&mut runner.state, false, &crate::run::home())
16192 .await
16193 .unwrap();
16194 runner.state = RunState::load(&runner.state.id).unwrap();
16195 assert!(runner.state.candidates[0].folded);
16196 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16197 assert_eq!(
16198 git::worktree_holding(&repo, &candidate.branch)
16199 .await
16200 .unwrap(),
16201 None
16202 );
16203 assert!(!runner.state.candidates[1].folded);
16204 assert!(winner.worktree.exists());
16205 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16206 }
16207 }
16208
16209 #[tokio::test]
16210 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16211 ask_test_home();
16212 let tmp = tempfile::tempdir().unwrap();
16213 let repo = tmp.path().join("repo");
16214 std::fs::create_dir(&repo).unwrap();
16215 init_repo(&repo);
16216 let mut runner = runner_at(RunStatus::Implementing);
16217 runner.state.repo = repo.clone();
16218 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16219 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16220 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16221 let branch = format!("candidate-{label}");
16222 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16223 git::worktree_add_branch(&repo, &wt, &branch, "main")
16224 .await
16225 .unwrap();
16226 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16227 git::git(&wt, &["add", "README.md"]).await.unwrap();
16228 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16229 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16230 runner.state.candidates.push(Candidate {
16231 index,
16232 label,
16233 agent: "alpha".to_owned(),
16234 branch,
16235 worktree: wt,
16236 summary: "completed implementation".to_owned(),
16237 stat: String::new(),
16238 files: 0,
16239 commits: 0,
16240 empty: false,
16241 failed: None,
16242 verified_noop: None,
16243 duration_ms: 42,
16244 folded: false,
16245 });
16246 runner
16247 .state
16248 .candidate_preservation
16249 .insert(label, "awaiting persistence".to_owned());
16250 }
16251 let wt = runner.state.candidates[0].worktree.clone();
16252 let index_lock = git::git(
16253 &wt,
16254 &[
16255 "rev-parse",
16256 "--path-format=absolute",
16257 "--git-path",
16258 "index.lock",
16259 ],
16260 )
16261 .await
16262 .unwrap();
16263 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16264 runner.state.save().unwrap();
16265 assert!(runner.persist_candidates().await.is_err());
16266 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16267 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16268 assert_eq!(
16269 runner.state.candidates[1].commits, 1,
16270 "other results are still collected"
16271 );
16272 assert!(
16273 !runner.state.candidates[0].empty,
16274 "failed rescue is not no change"
16275 );
16276 assert_eq!(
16277 runner.state.candidates[0].summary,
16278 "completed implementation"
16279 );
16280 assert!(runner.after_implement().is_err());
16281 let home = tmp.path().join("fold-home");
16282 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16283 assert!(wt.join("new-file").exists());
16284 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16285 assert!(recovery.join("staged.patch").exists());
16286 assert!(recovery.join("unstaged.patch").exists());
16287 assert_eq!(
16288 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16289 "untracked edits\n"
16290 );
16291 runner.state.begin_pass();
16292 runner.state.end_pass_parked(ParkCause::Upgrade {
16293 from: "1.0".into(),
16294 to: Some("1.1".into()),
16295 });
16296 runner.state.save().unwrap();
16297 runner.state = RunState::load(&runner.state.id).unwrap();
16298 assert!(matches!(
16299 runner.state.passes.last().unwrap().end,
16300 Some(crate::run::PassEnd::Parked {
16301 cause: ParkCause::Upgrade { .. }
16302 })
16303 ));
16304 std::fs::remove_file(index_lock).unwrap();
16305 runner.implement().await.unwrap();
16307 assert!(runner.state.candidate_preservation.is_empty());
16308 assert!(
16309 runner
16310 .state
16311 .candidates
16312 .iter()
16313 .all(|c| c.commits == 1 && !c.empty)
16314 );
16315 assert!(git::is_clean(&wt).await.unwrap());
16316 assert!(
16317 crate::run::read_artifact(&runner.state, "cand-A.patch")
16318 .unwrap()
16319 .contains("later tracked edits")
16320 );
16321 }
16322
16323 #[tokio::test]
16324 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16325 ask_test_home();
16326 let tmp = tempfile::tempdir().unwrap();
16327 let repo = tmp.path().join("repo");
16328 std::fs::create_dir(&repo).unwrap();
16329 init_repo(&repo);
16330 let wt = tmp.path().join("candidate");
16331 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16332 .await
16333 .unwrap();
16334 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16335 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16336 assert!(wt.join("uncommitted").exists());
16337 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16338 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16339 let mut runner = runner_at(RunStatus::Implementing);
16340 runner.state.repo = repo.clone();
16341 runner.state.base_commit = "missing-base".to_owned();
16342 runner.state.candidates.push(Candidate {
16343 index: 0,
16344 label: 'A',
16345 agent: "alpha".into(),
16346 branch: "candidate".into(),
16347 worktree: wt.clone(),
16348 summary: "completed".into(),
16349 stat: String::new(),
16350 files: 0,
16351 commits: 0,
16352 empty: false,
16353 failed: None,
16354 verified_noop: None,
16355 duration_ms: 1,
16356 folded: false,
16357 });
16358 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16359 assert!(
16360 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16361 .await
16362 .is_err()
16363 );
16364 assert!(!runner.state.candidates[0].folded);
16365 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16366 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16367 runner
16368 .state
16369 .candidate_preservation
16370 .insert('A', "pending".into());
16371 assert!(runner.persist_candidates().await.is_err());
16372 assert!(!runner.state.candidates[0].empty);
16373 assert!(!runner.state.candidates[0].folded);
16374 assert!(wt.exists());
16375 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16376 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16377 std::fs::create_dir_all(&patch_path).unwrap();
16378 assert!(
16379 runner.persist_candidates().await.is_err(),
16380 "artifact failure stays pending"
16381 );
16382 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16383 assert!(!runner.state.candidates[0].empty);
16384 std::fs::remove_dir(patch_path).unwrap();
16385 runner.persist_candidates().await.unwrap();
16386 assert!(
16387 runner.state.candidates[0].empty,
16388 "only successful inspection confirms no change"
16389 );
16390 }
16391
16392 fn init_repo(dir: &Path) {
16393 let run = |args: &[&str]| {
16394 let out = std::process::Command::new("git")
16395 .args(args)
16396 .current_dir(dir)
16397 .quiet()
16398 .output()
16399 .expect("spawn git");
16400 assert!(
16401 out.status.success(),
16402 "git {args:?} failed: {}",
16403 String::from_utf8_lossy(&out.stderr)
16404 );
16405 };
16406 run(&["init", "-b", "main"]);
16407 run(&["config", "user.name", "magi test"]);
16408 run(&["config", "user.email", "magi@example.com"]);
16409 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16410 run(&["add", "-A"]);
16411 run(&["commit", "-m", "init"]);
16412 }
16413
16414 fn ask_test_home() {
16422 crate::run::pin_test_home();
16423 }
16424
16425 #[test]
16426 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16427 ask_test_home();
16428 let mut runner = runner_at(RunStatus::Prep);
16429 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16430 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16433 let error = guard
16434 .finish(
16435 &mut runner.state,
16436 Err(
16437 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16438 .into(),
16439 ),
16440 )
16441 .unwrap_err();
16442 assert_eq!(
16443 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16444 std::io::ErrorKind::PermissionDenied
16445 );
16446 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16447 let message = format!("{error:#}");
16448 assert!(message.contains("original refusal"));
16449 assert!(!message.contains("could not save terminal"));
16450 assert_eq!(
16451 error.chain().count(),
16452 2,
16453 "only startup attribution wraps the original error"
16454 );
16455 let saved = RunState::load(&runner.state.id).unwrap();
16456 assert_eq!(saved.status, RunStatus::Blocked);
16457 assert!(!saved.startup.unwrap().complete);
16458 }
16459
16460 #[test]
16461 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16462 ask_test_home();
16463 let mut runner = runner_at(RunStatus::Prep);
16464 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16465 let id = runner.state.id.clone();
16466 drop(guard);
16467 let saved = RunState::load(&id).unwrap();
16468 assert_eq!(saved.status, RunStatus::Blocked);
16469 assert!(!saved.startup.as_ref().unwrap().complete);
16470 assert!(
16471 saved
16472 .startup
16473 .unwrap()
16474 .failure
16475 .unwrap()
16476 .contains("interrupted")
16477 );
16478 assert!(Runner::resume(&id).is_err());
16479 }
16480
16481 #[test]
16482 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16483 ask_test_home();
16484 let mut runner = runner_at(RunStatus::Prep);
16485 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16486 let error = guard
16487 .finish(
16488 &mut runner.state,
16489 Err(
16490 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16491 .into(),
16492 ),
16493 )
16494 .unwrap_err();
16495 assert_eq!(
16496 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16497 std::io::ErrorKind::PermissionDenied
16498 );
16499 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16500 let mut task = crate::queue::Task::new(
16501 "review".into(),
16502 "criteria".into(),
16503 runner.state.repo.clone(),
16504 crate::queue::Source::Human,
16505 );
16506 task.record_startup_failure(failure.clone());
16507 task.record_startup_failure(failure.clone());
16508 assert_eq!(task.startup_failures.len(), 1);
16509 assert!(task.runs.is_empty());
16510 assert_eq!(task.attempts, 0);
16511 }
16512
16513 #[cfg(unix)]
16514 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16515 ask_test_home();
16516 let tmp = tempfile::tempdir().unwrap();
16517 let repo = tmp.path().join("repo");
16518 std::fs::create_dir(&repo).unwrap();
16519 init_repo(&repo);
16520 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16521 let origin = tmp.path().join("origin.git");
16522 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16523 .await
16524 .unwrap();
16525 git::git(
16526 &repo,
16527 &["remote", "add", "origin", origin.to_str().unwrap()],
16528 )
16529 .await
16530 .unwrap();
16531 git::git(&repo, &["checkout", "-b", "fix/retained"])
16532 .await
16533 .unwrap();
16534 std::fs::write(repo.join("retained"), "retained change").unwrap();
16535 git::git(&repo, &["add", "."]).await.unwrap();
16536 git::git(&repo, &["commit", "-m", "retained implementation"])
16537 .await
16538 .unwrap();
16539 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16540 git::git(&repo, &["checkout", "main"]).await.unwrap();
16541 std::fs::write(repo.join("base-change"), "new base").unwrap();
16542 git::git(&repo, &["add", "."]).await.unwrap();
16543 git::git(&repo, &["commit", "-m", "base moved"])
16544 .await
16545 .unwrap();
16546 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16547 git::git(&repo, &["checkout", "fix/retained"])
16548 .await
16549 .unwrap();
16550 let mut runner = runner_at(RunStatus::Voting);
16551 runner.state.repo = repo.clone();
16552 runner.state.base_commit = base;
16553 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16554 runner.state.config.disk.min_free_bytes = 0;
16555 runner.state.config.graph.review_rounds = 1;
16556 runner.state.config.verify.e2e = vec!["true".into()];
16557 runner.state.config.verify.gate = vec!["true".into()];
16558 runner.state.candidates = vec![Candidate {
16559 index: 0,
16560 label: 'A',
16561 agent: "fixture".into(),
16562 branch: "fix/retained".into(),
16563 worktree: repo.clone(),
16564 summary: String::new(),
16565 stat: String::new(),
16566 files: 1,
16567 commits: 1,
16568 empty: false,
16569 failed: None,
16570 verified_noop: None,
16571 duration_ms: 0,
16572 folded: false,
16573 }];
16574 runner.state.tally = Some(Tally {
16575 first_choice: BTreeMap::new(),
16576 borda: BTreeMap::new(),
16577 winner: 'A',
16578 rankings: 0,
16579 unanimous_initial: true,
16580 deliberated: false,
16581 changed_votes: 0,
16582 unanimous_final: true,
16583 tie_break: None,
16584 judges: 0,
16585 present: 0,
16586 quorum: 0,
16587 met_quorum: true,
16588 uncontested: Some("review-only".into()),
16589 });
16590 let mut task = crate::queue::Task::new(
16591 "fixture".into(),
16592 "fixture".into(),
16593 repo,
16594 crate::queue::Source::Human,
16595 );
16596 task.link_run(&runner.state.id);
16597 crate::queue::Queue::open().put(&mut task).unwrap();
16598 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16599 runner.state.base_sync = Some(BaseSync {
16600 tip: String::new(),
16601 behind: 1,
16602 attempts: BASE_SYNC_ROUNDS,
16603 conflict: None,
16604 already_in: None,
16605 });
16606 runner.sync_to_base().await.unwrap();
16607 assert_eq!(runner.state.status, RunStatus::Blocked);
16608 (tmp, runner, head, task.id)
16609 }
16610
16611 #[cfg(unix)]
16612 #[tokio::test]
16613 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16614 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16615 if std::env::var_os(CHILD).is_none() {
16616 let home = tempfile::tempdir().unwrap();
16617 let output = std::process::Command::new(std::env::current_exe().unwrap())
16618 .args([
16619 "--exact",
16620 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16621 "--nocapture",
16622 ])
16623 .env(CHILD, "1")
16624 .env("MAGI_HOME", home.path())
16625 .env("MAGI_WORKER_HOME", home.path())
16626 .env("MAGI_WORKER_ID", "abcdef")
16627 .output()
16628 .unwrap();
16629 assert!(
16630 output.status.success()
16631 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16632 "{}\n{}",
16633 String::from_utf8_lossy(&output.stdout),
16634 String::from_utf8_lossy(&output.stderr)
16635 );
16636 return;
16637 }
16638 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16639 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16640 let home = crate::run::home();
16641 let queue = crate::queue::Queue::open();
16642 let mut owner = queue.get(&task).unwrap();
16643 owner.release_operator("test-release", None);
16644 owner.start(runner.state.id.clone());
16645 queue.put(&mut owner).unwrap();
16646 runner.state.driver_exited = true;
16647 runner.state.task_instruction = Some(owner.instruction.clone());
16648 runner.state.save().unwrap();
16649 let _claim = queue.claim(&task).unwrap();
16650 let reserve = |id: &str| {
16651 let directory = home.join("workers/jobs").join(id);
16652 std::fs::create_dir_all(&directory).unwrap();
16653 let job = crate::worker::Job {
16654 id: id.into(),
16655 sequence: 1,
16656 kind: "task".into(),
16657 key: format!("task:{task}"),
16658 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16659 generation: "fixture-generation".into(),
16660 version: None,
16661 status: crate::worker::Status::Running,
16662 result: None,
16663 pid: Some(std::process::id()),
16664 created_at: 0,
16665 finished_at: None,
16666 };
16667 std::fs::write(
16668 directory.join("job.json"),
16669 serde_json::to_vec(&job).unwrap(),
16670 )
16671 .unwrap();
16672 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16673 };
16674 let _own = reserve("abcdef");
16675 let other = reserve("fedcba");
16676 assert!(
16677 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16678 .await
16679 .is_err()
16680 );
16681 assert!(
16682 RunState::load(&runner.state.id)
16683 .unwrap()
16684 .base_sync_recovery
16685 .authorizations
16686 .is_empty()
16687 );
16688 drop(other);
16689 assert!(crate::daemon::is_working_on(
16690 &home,
16691 &runner.state.id,
16692 Timestamp::now()
16693 ));
16694 assert!(
16695 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16696 );
16697 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16698 .await
16699 .unwrap();
16700 assert!(
16701 RunState::load(&runner.state.id)
16702 .unwrap()
16703 .base_recovery_pending()
16704 );
16705 }
16706
16707 #[cfg(unix)]
16708 #[tokio::test]
16709 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16710 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16711 let home = crate::run::home();
16712 runner.state.driver_pid = Some(std::process::id());
16713 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16714 runner.state.driver_exited = false;
16715 runner.state.begin_pass();
16716 runner.state.save().unwrap();
16717 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16718 driver.record(&runner.state).unwrap();
16719 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16720 drop(driver);
16722 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16723 assert_eq!(
16724 runner.state.liveness_under(&home, false),
16725 crate::run::Liveness::Dead
16726 );
16727 let queue = crate::queue::Queue::open();
16728 let _claim = queue.claim(&task).unwrap();
16729 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16730 .await
16731 .unwrap();
16732 let disk = RunState::load(&runner.state.id).unwrap();
16733 assert!(!disk.driver_exited);
16734 assert!(disk.base_recovery_pending());
16735 }
16736
16737 #[cfg(unix)]
16738 #[tokio::test]
16739 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16740 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16741 if std::env::var_os(CHILD).is_none() {
16742 use std::os::unix::fs::PermissionsExt;
16743 let tmp = tempfile::tempdir().unwrap();
16744 let gh = tmp.path().join("gh");
16745 std::fs::write(&gh, r#"#!/bin/sh
16746case "$1 $2" in
16747 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16748 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16749 *) exit 2 ;;
16750esac
16751"#).unwrap();
16752 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16753 let mut paths = vec![tmp.path().to_path_buf()];
16754 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16755 let output = std::process::Command::new(std::env::current_exe().unwrap())
16756 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16757 .env(CHILD, "1")
16758 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16759 .env("PATH", std::env::join_paths(paths).unwrap())
16760 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16761 .output().unwrap();
16762 assert!(
16763 output.status.success()
16764 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16765 "{}\n{}",
16766 String::from_utf8_lossy(&output.stdout),
16767 String::from_utf8_lossy(&output.stderr)
16768 );
16769 return;
16770 }
16771 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16772 let repo = runner.state.repo.clone();
16773 git::git(&repo, &["checkout", "main"]).await.unwrap();
16774 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16775 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16776 git::git(&repo, &["checkout", "fix/retained"])
16777 .await
16778 .unwrap();
16779 runner.state.pr = Some(crate::run::PrRecord {
16780 url: "https://example.test/project/pull/1".into(),
16781 number: 1,
16782 state: "open".into(),
16783 checks: "unknown".into(),
16784 round: 0,
16785 rounds: 1,
16786 red_at_merge: Vec::new(),
16787 });
16788 runner.sync_to_base().await.unwrap();
16789 assert_eq!(runner.state.status, RunStatus::Blocked);
16790 assert_eq!(
16791 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16792 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16793 );
16794 assert!(runner.state.base_sync_closure_pending());
16795 assert!(crate::handover::can_resume(&runner.state));
16796 let queue = crate::queue::Queue::open();
16797 assert!(!crate::base_sync_recovery::held_task(
16798 &queue,
16799 &queue.get(&task).unwrap()
16800 ));
16801 assert!(runner.state.authorize_base_sync(&task).is_err());
16802 runner.state.base_sync_recovery.stop = None;
16804 runner.state.status = RunStatus::Voting;
16805 runner.state.gate_ran = true;
16806 runner.state.config.graph.review_rounds = 0;
16807 runner.state.save().unwrap();
16808 assert!(crate::handover::can_resume(&runner.state));
16809 runner.state = RunState::load(&runner.state.id).unwrap();
16810 runner.execute_graph().await.unwrap();
16811 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16812 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16813 assert_eq!(
16814 runner.state.base_sync.as_ref().unwrap().attempts,
16815 BASE_SYNC_ROUNDS
16816 );
16817 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16818 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16819 assert!(runner.state.reviews.is_empty());
16820 assert!(runner.state.gate.is_empty());
16821 }
16822
16823 #[cfg(unix)]
16824 #[tokio::test]
16825 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16826 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16827 runner.state.driver_exited = true;
16828 runner.state.save().unwrap();
16829 let home = crate::run::home();
16830 let queue = crate::queue::Queue::open();
16831 let _claim = queue.claim(&task).unwrap();
16832 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16833 assert!(
16834 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16835 .await
16836 .is_err()
16837 );
16838 drop(lease);
16839 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16840 assert!(
16841 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16842 .await
16843 .is_err()
16844 );
16845 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16846 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16847 .await
16848 .unwrap();
16849 let disk = RunState::load(&runner.state.id).unwrap();
16850 assert!(disk.base_recovery_pending());
16851 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16852 assert!(
16853 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16854 .await
16855 .is_err()
16856 );
16857 assert_eq!(
16858 RunState::load(&runner.state.id)
16859 .unwrap()
16860 .base_sync_recovery
16861 .authorizations
16862 .len(),
16863 1
16864 );
16865 let mut owner = queue.get(&task).unwrap();
16866 owner.runs.push("20260101-000000-ffff".into());
16867 queue.put(&mut owner).unwrap();
16868 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16869 .await
16870 .unwrap_err();
16871 assert!(error.to_string().contains("run task ownership changed"));
16872 }
16873
16874 #[cfg(unix)]
16875 #[tokio::test]
16876 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16877 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16878 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16879 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16880 legacy["status"] = serde_json::json!("voting");
16881 runner.state = serde_json::from_value(legacy).unwrap();
16882 runner.state.save().unwrap();
16883 for _ in 0..3 {
16884 runner.execute_graph().await.unwrap();
16885 assert_eq!(runner.state.status, RunStatus::Blocked);
16886 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16887 assert!(!crate::handover::can_resume(&runner.state));
16888 assert_eq!(
16889 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16890 head
16891 );
16892 assert!(runner.state.reviews.is_empty());
16893 assert!(!runner.state.gate_ran);
16894 }
16895 let notice = crate::notices::run_ended(&runner.state).unwrap();
16896 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16897 runner.state.begin_pass();
16898 runner.state.status = RunStatus::Voting;
16899 runner.deliberate().await.unwrap();
16900 runner.vote().await.unwrap();
16901 runner.finish_after_tally().await.unwrap();
16902 assert_eq!(runner.state.status, RunStatus::Blocked);
16903 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16904 assert_eq!(
16905 crate::notices::run_ended(&runner.state).unwrap().since,
16906 notice.since
16907 );
16908 }
16909
16910 #[cfg(unix)]
16911 #[tokio::test]
16912 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16913 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16914 for detail in [
16915 "fix/retained conflicts with origin/main and did not rebase: conflict",
16916 "rebased fix/retained locally but origin refused the push",
16917 "unknown failure",
16918 ] {
16919 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16920 runner.state.record_base_sync_stop(&head);
16921 assert!(runner.state.authorize_base_sync(&task).is_err());
16922 runner.state.status = RunStatus::Voting;
16923 runner.execute_graph().await.unwrap();
16924 assert_eq!(runner.state.status, RunStatus::Blocked);
16925 assert!(!crate::handover::can_resume(&runner.state));
16926 assert_eq!(
16927 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16928 head
16929 );
16930 }
16931 }
16932
16933 #[cfg(unix)]
16934 #[tokio::test]
16935 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16936 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16937 runner.state.authorize_base_sync(&task).unwrap();
16938 assert!(runner.state.consume_base_recovery(&head));
16939 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16940 runner.state.reserve_base_operation(&head, &tip);
16941 runner.state.save().unwrap();
16942 runner.state = RunState::load(&runner.state.id).unwrap();
16943 assert!(!runner.state.consume_base_recovery(&head));
16944 runner.sync_to_base().await.unwrap();
16945 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16946 assert_eq!(runner.state.base_episode_used(), 1);
16947 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16948 assert!(
16949 runner
16950 .state
16951 .base_sync_recovery
16952 .operations
16953 .last()
16954 .unwrap()
16955 .finished
16956 );
16957 assert!(!runner.state.base_sync_blocked());
16958 runner.sync_to_base().await.unwrap();
16959 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16960 }
16961
16962 #[cfg(unix)]
16963 #[tokio::test]
16964 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16965 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16966 let original = runner
16968 .state
16969 .base_sync
16970 .as_ref()
16971 .unwrap()
16972 .conflict
16973 .clone()
16974 .unwrap();
16975 runner.state.base_sync.as_mut().unwrap().conflict = Some(
16976 original
16977 .replace("synchronization(s)", "rebase(s)")
16978 .replace("synchronizing", "rebasing"),
16979 );
16980 runner.state.base_sync_recovery = Default::default();
16981 runner.state.record_base_sync_stop(&head);
16982 assert_eq!(
16983 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16984 crate::base_sync_recovery::StopKind::Exhausted
16985 );
16986 runner.state.authorize_base_sync(&task).unwrap();
16987 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16988 let auth = legacy["base_sync_recovery"]["authorizations"][0]
16989 .as_object_mut()
16990 .unwrap();
16991 auth.remove("interval");
16992 auth.remove("scope");
16993 runner.state = serde_json::from_value(legacy).unwrap();
16994 assert!(runner.state.consume_base_recovery(&head));
16995 assert_eq!(runner.state.base_episode_used(), 3);
16996 runner.sync_to_base().await.unwrap();
16997 assert_eq!(runner.state.base_episode_used(), 4);
16998 runner.state.resource_block = Some(crate::run::ResourceBlock {
16999 node: "gate".into(),
17000 kind: "cache".into(),
17001 detail: "legacy wait".into(),
17002 at: Timestamp::now(),
17003 cause: crate::run::ResourceBlockCause::Unrecorded,
17004 });
17005 runner.state.save().unwrap();
17006 assert!(
17007 runner
17008 .state
17009 .base_sync_recovery
17010 .episodes
17011 .last()
17012 .unwrap()
17013 .boundary
17014 .is_none()
17015 );
17016 runner.state.resource_block = None;
17017 advance_base_fixture(&runner).await;
17018 runner.sync_to_base().await.unwrap();
17019 assert!(runner.state.base_sync_blocked());
17020 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17021 }
17022
17023 #[cfg(unix)]
17024 async fn advance_base_fixture(runner: &Runner) {
17025 let repo = &runner.state.repo;
17026 for args in [
17027 vec!["checkout", "main"],
17028 vec!["commit", "--allow-empty", "-m", "base advanced"],
17029 vec!["push", "origin", "main"],
17030 vec!["checkout", "fix/retained"],
17031 ] {
17032 git::git(repo, &args).await.unwrap();
17033 }
17034 }
17035
17036 #[cfg(unix)]
17037 fn fresh_base_interval_fixture(runner: &mut Runner) {
17038 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17039 runner.state.base_sync.as_mut().unwrap().conflict = None;
17040 runner.state.base_sync_recovery = Default::default();
17041 runner.state.status = RunStatus::Voting;
17042 }
17043
17044 #[cfg(unix)]
17045 #[tokio::test]
17046 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17047 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17048 fresh_base_interval_fixture(&mut runner);
17049 runner.state.begin_pass();
17050 runner.sync_to_base().await.unwrap();
17051 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17052 runner.state.resource_block = Some(crate::run::ResourceBlock {
17053 node: "gate".into(),
17054 kind: "build cache".into(),
17055 detail: "busy".into(),
17056 at: Timestamp::now(),
17057 cause: crate::run::ResourceBlockCause::Timeout {
17058 waited_secs: 1,
17059 limit_secs: 1,
17060 },
17061 });
17062 runner.state.save().unwrap();
17063 let queue = crate::queue::Queue::open();
17064 let mut owner = queue.get(&task).unwrap();
17065 runner.state.task_instruction = Some(owner.instruction.clone());
17066 owner.release_operator("fixture-owner", None);
17067 queue.put(&mut owner).unwrap();
17068 let approval = owner.resumption.clone().unwrap();
17069 runner.state.resource_block = None;
17070 runner.state.save().unwrap();
17071 runner.state = RunState::load(&runner.state.id).unwrap();
17072 advance_base_fixture(&runner).await;
17073 runner.sync_to_base().await.unwrap();
17074 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17075 assert_eq!(runner.state.base_episode_used(), 1);
17076 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17077 assert!(
17078 runner.state.base_sync_recovery.episodes[0]
17079 .boundary
17080 .as_ref()
17081 .unwrap()
17082 .consumed
17083 );
17084 assert!(
17085 !runner
17086 .state
17087 .renew_base_episode(&task, &approval, &head, "base")
17088 );
17089 for _ in 0..3 {
17090 owner.release_operator("fixture-owner", None);
17091 queue.put(&mut owner).unwrap();
17092 runner.state = RunState::load(&runner.state.id).unwrap();
17093 runner.state.begin_pass();
17094 runner.sync_to_base().await.unwrap();
17095 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17096 assert_eq!(runner.state.base_episode_used(), 1);
17097 }
17098 runner.state.resource_block = Some(crate::run::ResourceBlock {
17099 node: "gate".into(),
17100 kind: "disk pressure".into(),
17101 detail: "free space required".into(),
17102 at: Timestamp::now(),
17103 cause: crate::run::ResourceBlockCause::Other,
17104 });
17105 runner.state.end_pass_parked(ParkCause::Interrupt {
17106 reason: "disk pressure".into(),
17107 });
17108 runner.state.save().unwrap();
17109 owner.release_operator("fixture-owner-after-disk", None);
17110 queue.put(&mut owner).unwrap();
17111 runner.state.resource_block = None;
17112 advance_base_fixture(&runner).await;
17113 runner.sync_to_base().await.unwrap();
17114 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17115 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17116 assert_eq!(runner.state.base_episode_used(), 1);
17117 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17118 }
17119
17120 #[cfg(unix)]
17121 #[tokio::test]
17122 async fn base_recovery_continuous_advancement_stops_at_four() {
17123 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17124 fresh_base_interval_fixture(&mut runner);
17125 for spent in 1..=BASE_SYNC_ROUNDS {
17126 runner.sync_to_base().await.unwrap();
17127 assert!(!runner.state.base_sync_blocked());
17128 assert_eq!(runner.state.base_episode_used(), spent);
17129 advance_base_fixture(&runner).await;
17130 }
17131 runner.sync_to_base().await.unwrap();
17132 assert_eq!(
17133 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17134 crate::base_sync_recovery::StopKind::Exhausted
17135 );
17136 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17137 assert!(stop.unverified_sync);
17138 let action = runner.state.base_sync_next_action();
17139 assert!(action.contains("never reached its gate"));
17140 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17141 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17142 runner.state = RunState::load(&runner.state.id).unwrap();
17143 runner.sync_to_base().await.unwrap();
17144 assert_eq!(runner.state.base_episode_used(), 4);
17145 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17146 }
17147
17148 #[cfg(unix)]
17149 #[tokio::test]
17150 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17151 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17152 runner.state.ensure_base_episode();
17153 runner.state.record_base_progress(&head);
17155 runner.state.base_sync_recovery.stop = None;
17156 runner.state.record_base_sync_stop(&head);
17157 assert!(
17158 !runner
17159 .state
17160 .base_sync_recovery
17161 .stop
17162 .as_ref()
17163 .unwrap()
17164 .unverified_sync
17165 );
17166
17167 runner.state.finish_base_operation(Some(&head));
17168 runner.state.gate_ran = true;
17170 runner.state.gate_head = Some("older-head".into());
17171 runner.state.base_sync_recovery.stop = None;
17172 runner.state.record_base_sync_stop(&head);
17173 runner.state.save().unwrap();
17174 runner.state = RunState::load(&runner.state.id).unwrap();
17175 assert!(
17176 runner
17177 .state
17178 .base_sync_recovery
17179 .stop
17180 .as_ref()
17181 .unwrap()
17182 .unverified_sync
17183 );
17184
17185 runner.state.gate_head = Some(head.clone());
17186 runner.state.base_sync_recovery.stop = None;
17187 runner.state.record_base_sync_stop(&head);
17188 assert!(
17189 !runner
17190 .state
17191 .base_sync_recovery
17192 .stop
17193 .as_ref()
17194 .unwrap()
17195 .unverified_sync
17196 );
17197 assert!(
17198 !runner
17199 .state
17200 .base_sync_next_action()
17201 .contains("never reached its gate")
17202 );
17203
17204 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17206 legacy["base_sync_recovery"]["stop"]
17207 .as_object_mut()
17208 .unwrap()
17209 .remove("unverified_sync");
17210 legacy["base_sync_recovery"]["episodes"][0]
17211 .as_object_mut()
17212 .unwrap()
17213 .remove("last_sync_head");
17214 runner.state = serde_json::from_value(legacy).unwrap();
17215 assert!(
17216 !runner
17217 .state
17218 .base_sync_recovery
17219 .stop
17220 .as_ref()
17221 .unwrap()
17222 .unverified_sync
17223 );
17224 assert!(
17225 runner.state.base_sync_recovery.episodes[0]
17226 .last_sync_head
17227 .is_none()
17228 );
17229 }
17230
17231 #[cfg(unix)]
17232 #[tokio::test]
17233 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17234 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17235 runner.state.authorize_base_sync(&task).unwrap();
17236 assert!(
17237 runner
17238 .state
17239 .base_sync_next_action()
17240 .contains("Approval recorded")
17241 );
17242 assert!(
17243 runner
17244 .state
17245 .base_sync_next_action()
17246 .contains("4/4 in episode 0, 4 lifetime")
17247 );
17248 assert!(runner.state.consume_base_recovery(&head));
17249 assert!(
17250 runner
17251 .state
17252 .base_sync_next_action()
17253 .contains("0/4 in episode 1, 4 lifetime")
17254 );
17255 }
17256
17257 #[cfg(unix)]
17258 #[tokio::test]
17259 async fn base_recovery_changed_head_invalidates_without_spending() {
17260 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17261 runner.state.authorize_base_sync(&task).unwrap();
17262 git::git(
17263 &runner.state.repo,
17264 &["commit", "--allow-empty", "-m", "external head change"],
17265 )
17266 .await
17267 .unwrap();
17268 runner.sync_to_base().await.unwrap();
17269 let a = runner
17270 .state
17271 .base_sync_recovery
17272 .authorizations
17273 .last()
17274 .unwrap();
17275 assert!(a.consumed);
17276 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17277 runner.state = RunState::load(&runner.state.id).unwrap();
17278 assert!(
17279 runner
17280 .state
17281 .base_sync_next_action()
17282 .contains("Approval invalidated")
17283 );
17284 assert!(!runner.state.base_recovery_pending());
17285 assert!(runner.state.base_sync_blocked());
17286 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17287 }
17288
17289 #[cfg(unix)]
17290 #[tokio::test]
17291 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17292 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17293 runner.state.driver_exited = true;
17294 runner.state.save().unwrap();
17295 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17296 .await
17297 .unwrap();
17298 runner.state = RunState::load(&runner.state.id).unwrap();
17299 advance_base_fixture(&runner).await;
17300 runner.sync_to_base().await.unwrap();
17301 let a = runner
17302 .state
17303 .base_sync_recovery
17304 .authorizations
17305 .last()
17306 .unwrap();
17307 assert!(
17308 a.consumed
17309 && a.invalidated
17310 .as_deref()
17311 .unwrap()
17312 .contains("fetched base changed")
17313 );
17314 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17315 assert_eq!(
17316 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17317 head
17318 );
17319 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17320 .await
17321 .unwrap();
17322 runner.state = RunState::load(&runner.state.id).unwrap();
17323 runner.sync_to_base().await.unwrap();
17324 assert!(!runner.state.base_sync_blocked());
17325 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17326 }
17327
17328 #[cfg(unix)]
17329 #[tokio::test]
17330 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17331 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17332 runner.state.authorize_base_sync(&task).unwrap();
17333 assert!(runner.state.consume_base_recovery(&head));
17334 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17335 runner.state.reserve_base_operation(&head, &tip);
17336 runner.state.save().unwrap();
17337 let scratch = runner.state.dir().join("base-sync");
17338 assert_eq!(
17339 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17340 .await
17341 .unwrap(),
17342 crate::rebase::Rebased::Applied
17343 );
17344 runner.state = RunState::load(&runner.state.id).unwrap();
17345 runner.sync_to_base().await.unwrap();
17346 assert!(
17347 runner
17348 .state
17349 .base_sync_recovery
17350 .operations
17351 .last()
17352 .unwrap()
17353 .finished
17354 );
17355 assert_eq!(runner.state.base_episode_used(), 1);
17356 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17357 assert!(!runner.state.base_sync_blocked());
17358 }
17359
17360 #[cfg(unix)]
17361 #[tokio::test]
17362 async fn base_recovery_crash_preserves_untracked_files() {
17363 for ignored in [false, true] {
17364 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17365 runner.state.authorize_base_sync(&task).unwrap();
17366 assert!(runner.state.consume_base_recovery(&head));
17367 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17368 runner.state.reserve_base_operation(&head, &tip);
17369 runner.state.save().unwrap();
17370 let scratch = runner.state.dir().join("base-sync");
17371 assert_eq!(
17372 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17373 .await
17374 .unwrap(),
17375 crate::rebase::Rebased::Applied
17376 );
17377 let repo = &runner.state.repo;
17378 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17379 assert_ne!(result, head);
17380 if ignored {
17381 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17382 .await
17383 .unwrap();
17384 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17385 }
17386 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17387 assert!(
17388 git::git_raw(repo, &["diff", "--quiet", &head])
17389 .await
17390 .unwrap()
17391 .ok()
17392 );
17393 runner.state = RunState::load(&runner.state.id).unwrap();
17395 runner.sync_to_base().await.unwrap();
17396 assert!(runner.state.base_sync_blocked());
17397 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17398 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17399 assert_eq!(
17400 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17401 result
17402 );
17403 assert_eq!(
17404 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17405 "retained work\n"
17406 );
17407 assert!(
17408 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17409 .await
17410 .unwrap()
17411 .ok()
17412 );
17413 }
17414 }
17415
17416 #[cfg(unix)]
17417 #[tokio::test]
17418 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17419 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17420 runner.state.authorize_base_sync(&task).unwrap();
17421 assert!(runner.state.consume_base_recovery(&head));
17422 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17423 runner.state.reserve_base_operation(&head, &tip);
17424 runner.state.save().unwrap();
17425 let scratch = runner.state.dir().join("base-sync");
17426 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17427 .await
17428 .unwrap();
17429 git::sync_to_head(&runner.state.repo).await.unwrap();
17430 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17431 git::git(&runner.state.repo, &["add", "foreign"])
17432 .await
17433 .unwrap();
17434 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17435 .await
17436 .unwrap();
17437 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17438 runner.state = RunState::load(&runner.state.id).unwrap();
17439 runner.sync_to_base().await.unwrap();
17440 assert!(runner.state.base_sync_blocked());
17441 assert!(
17442 !runner
17443 .state
17444 .base_sync_recovery
17445 .operations
17446 .last()
17447 .unwrap()
17448 .finished
17449 );
17450 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17451 assert_eq!(
17452 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17453 foreign_head
17454 );
17455 }
17456
17457 #[cfg(unix)]
17458 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17459 {
17460 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17461 fresh_base_interval_fixture(&mut runner);
17462 let repo = &runner.state.repo;
17463 git::git(repo, &["checkout", "main"]).await.unwrap();
17464 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17465 git::git(repo, &["add", "retained"]).await.unwrap();
17466 git::git(repo, &["commit", "-m", "upstream retained change"])
17467 .await
17468 .unwrap();
17469 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17470 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17471 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17472 let scratch = runner.state.dir().join("base-sync");
17473 assert!(matches!(
17474 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17475 .await
17476 .unwrap(),
17477 git::RebaseStart::Conflicted(_)
17478 ));
17479 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17480 agent: "fixture".into(),
17481 paths: vec!["retained".into()],
17482 from: Some(head.clone()),
17483 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17484 index_outside: None,
17485 finished: false,
17486 error: None,
17487 launcher: None,
17488 infra: None,
17489 outage: None,
17490 });
17491 (tmp, runner, head, tip, scratch)
17492 }
17493
17494 #[cfg(unix)]
17495 #[tokio::test]
17496 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17497 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17498 std::fs::write(
17501 scratch.join("retained"),
17502 "upstream and retained resolution\n",
17503 )
17504 .unwrap();
17505 runner.state.resource_block = Some(crate::run::ResourceBlock {
17506 node: "rebase".into(),
17507 kind: "repository coordination".into(),
17508 detail: "busy".into(),
17509 at: Timestamp::now(),
17510 cause: crate::run::ResourceBlockCause::Other,
17511 });
17512 runner.state.save().unwrap();
17513 let newer = git::git(
17516 &runner.state.repo,
17517 &[
17518 "commit-tree",
17519 "main^{tree}",
17520 "-p",
17521 "main",
17522 "-m",
17523 "base advanced during wait",
17524 ],
17525 )
17526 .await
17527 .unwrap();
17528 git::git(
17529 &runner.state.repo,
17530 &[
17531 "push",
17532 "origin",
17533 &format!("{}:refs/heads/main", newer.trim()),
17534 ],
17535 )
17536 .await
17537 .unwrap();
17538 runner.state = RunState::load(&runner.state.id).unwrap();
17539 runner.state.resource_block = None;
17540 runner.sync_to_base().await.unwrap();
17541 assert!(!runner.state.base_sync_blocked());
17542 let op = &runner.state.base_sync_recovery.operations[0];
17543 assert_eq!(op.head, head);
17544 assert_eq!(op.tip, tip);
17545 assert!(op.finished);
17546 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17547 assert_eq!(runner.state.rebase_fixes.len(), 1);
17548 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17549 .await
17550 .unwrap();
17551 assert_ne!(latest, tip);
17552 runner.state = RunState::load(&runner.state.id).unwrap();
17553 runner.sync_to_base().await.unwrap();
17554 assert!(!runner.state.base_sync_blocked());
17555 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17556 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17557 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17558 }
17559
17560 #[cfg(unix)]
17561 #[tokio::test]
17562 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17563 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17564 runner.state.reserve_base_operation(&head, &tip);
17565 runner.state.save().unwrap();
17566 std::fs::write(
17567 scratch.join("retained"),
17568 "upstream and retained resolution\n",
17569 )
17570 .unwrap();
17571 git::git(&scratch, &["add", "retained"]).await.unwrap();
17572 git::git(
17573 &scratch,
17574 &["-c", "core.editor=true", "rebase", "--continue"],
17575 )
17576 .await
17577 .unwrap();
17578 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17579 assert!(
17580 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17581 .await
17582 .unwrap()
17583 .is_empty()
17584 );
17585 assert_eq!(
17586 crate::rebase::finish(
17587 &mut runner.state,
17588 &scratch,
17589 "fix/retained",
17590 &head,
17591 &tip,
17592 &["retained".into()],
17593 ""
17594 )
17595 .await
17596 .unwrap(),
17597 crate::rebase::Rebased::Applied
17598 );
17599 assert!(!scratch.exists());
17600 runner.state = RunState::load(&runner.state.id).unwrap();
17602 let op = &runner.state.base_sync_recovery.operations[0];
17603 assert!(!op.finished);
17604 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17605 runner.sync_to_base().await.unwrap();
17606 assert!(!runner.state.base_sync_blocked());
17607 assert!(runner.state.base_sync_recovery.operations[0].finished);
17608 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17609 assert_eq!(runner.state.rebase_fixes.len(), 1);
17610 let winner = runner.state.winner().unwrap().clone();
17611 let wait = runner.repository_wait();
17612 assert!(
17613 verify_current_head(&mut runner.state, &winner, &wait)
17614 .await
17615 .unwrap()
17616 );
17617 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17618 }
17619
17620 #[cfg(unix)]
17621 #[tokio::test]
17622 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17623 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17624 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17625 runner.state.reserve_base_operation(&head, &tip);
17626 runner.state.save().unwrap();
17627 std::fs::write(
17628 scratch.join("retained"),
17629 "upstream and retained resolution\n",
17630 )
17631 .unwrap();
17632 git::git(&scratch, &["add", "retained"]).await.unwrap();
17633 git::git(
17634 &scratch,
17635 &["-c", "core.editor=true", "rebase", "--continue"],
17636 )
17637 .await
17638 .unwrap();
17639 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17640 assert!(!git::rebase_in_progress(&scratch).await);
17641 assert!(
17642 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17643 .await
17644 .unwrap()
17645 .is_empty()
17646 );
17647 match damage {
17648 Some("record") => {
17649 let path =
17650 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17651 .await
17652 .unwrap();
17653 std::fs::write(scratch.join(path), "").unwrap();
17654 }
17655 Some("foreign") => {
17656 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17657 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17658 git::git(&scratch, &["commit", "-m", "foreign work"])
17659 .await
17660 .unwrap();
17661 }
17662 Some("dirty") => {
17663 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17664 }
17665 _ => {}
17666 }
17667 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17668 runner.state = RunState::load(&runner.state.id).unwrap();
17670 assert!(
17671 runner.state.base_sync_recovery.operations[0]
17672 .validated_head
17673 .is_none()
17674 );
17675 runner.sync_to_base().await.unwrap();
17676 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17677 assert_eq!(runner.state.base_episode_used(), 1);
17678 assert_eq!(runner.state.rebase_fixes.len(), 1);
17679 assert_eq!(
17680 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17681 retained_head
17682 );
17683 if damage.is_some() {
17684 assert!(runner.state.base_sync_blocked());
17685 assert!(scratch.exists());
17686 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17687 assert!(
17688 runner.state.base_sync_recovery.operations[0]
17689 .validated_head
17690 .is_none()
17691 );
17692 } else {
17693 assert!(!runner.state.base_sync_blocked());
17694 assert!(!scratch.exists());
17695 assert!(runner.state.base_sync_recovery.operations[0].finished);
17696 assert_eq!(
17697 runner.state.base_sync_recovery.operations[0]
17698 .validated_head
17699 .as_deref(),
17700 Some(result.as_str())
17701 );
17702 let winner = runner.state.winner().unwrap().clone();
17703 let wait = runner.repository_wait();
17704 assert!(
17705 verify_current_head(&mut runner.state, &winner, &wait)
17706 .await
17707 .unwrap()
17708 );
17709 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17710 runner.state = RunState::load(&runner.state.id).unwrap();
17711 runner.sync_to_base().await.unwrap();
17712 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17713 }
17714 }
17715 }
17716
17717 #[cfg(unix)]
17718 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17719 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17720 runner.state.prep_complete = Some(true);
17722 runner.state.base_sync = None;
17723 runner.state.base_sync_recovery = Default::default();
17724 runner.state.config.graph.gate_fix_rounds = 1;
17725 let mut review = review_round(true, 0, 1, 1, false, true);
17726 review.head = head.clone();
17727 review.verified_head = Some(head.clone());
17728 runner.state.reviews = vec![review];
17729 runner.state.gate = vec![CommandOutcome {
17730 command: "old gate".into(),
17731 code: Some(1),
17732 output_tail: "old failure".into(),
17733 duration_ms: 0,
17734 resource_blocked: false,
17735 }];
17736 runner.state.gate_head = Some(head.clone());
17737 runner.state.gate_ran = true;
17738 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17739 agent: "fixture".into(),
17740 failed: runner.state.gate.clone(),
17741 notes: "spent".into(),
17742 committed: false,
17743 error: None,
17744 }];
17745 runner.state.land_approval = Some(crate::run::LandApproval {
17746 question: "old approval".into(),
17747 head: head.clone(),
17748 });
17749 runner.state.status = RunStatus::Blocked;
17750 (tmp, runner, head)
17751 }
17752
17753 #[cfg(unix)]
17754 #[tokio::test]
17755 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17756 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17757 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17758 let phases = tmp.path().join("phases");
17759 runner.state.config.verify.e2e = vec![format!(
17760 "echo e2e >> '{}'; {}",
17761 phases.display(),
17762 if e2e_ok {
17763 "true"
17764 } else {
17765 "echo new-e2e-failure; false"
17766 }
17767 )];
17768 runner.state.config.verify.gate = vec![format!(
17769 "echo gate >> '{}'; {}",
17770 phases.display(),
17771 if gate_ok {
17772 "true"
17773 } else {
17774 "echo new-gate-failure; false"
17775 }
17776 )];
17777 runner.state.save().unwrap();
17778 runner.state = RunState::load(&runner.state.id).unwrap();
17779 runner.sync_to_base().await.unwrap();
17780 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17781 assert_ne!(head, old);
17782 runner.state = RunState::load(&runner.state.id).unwrap();
17784 runner.execute_graph().await.unwrap();
17785 assert_eq!(runner.state.gate_fixes.len(), 1);
17786 assert_eq!(runner.state.reviews.len(), 1);
17787 assert_eq!(runner.state.reviews[0].head, old);
17788 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17789 assert_eq!(runner.state.verification_history.len(), 1);
17790 let history = &runner.state.verification_history[0];
17791 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17792 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17793 assert!(history.archived_at.is_some());
17794 assert_eq!(history.fixes_spent, Some(1));
17795 assert_eq!(history.gate[0].output_tail, "old failure");
17796 let archive_events: Vec<_> = runner
17797 .state
17798 .events
17799 .iter()
17800 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17801 .collect();
17802 assert_eq!(archive_events.len(), 1);
17803 assert!(archive_events[0].message.contains(&short(&old)));
17804 assert!(archive_events[0].message.contains(&short(&head)));
17805 assert!(
17806 archive_events[0]
17807 .message
17808 .contains("1 gate-fix round(s) already spent stay spent")
17809 );
17810 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17811 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17812 if e2e_ok {
17813 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17814 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17815 } else {
17816 assert!(!runner.state.gate_ran);
17817 }
17818 assert_eq!(
17819 runner.state.status,
17820 if e2e_ok && gate_ok {
17821 RunStatus::Ready
17822 } else {
17823 RunStatus::Blocked
17824 }
17825 );
17826 if !e2e_ok || !gate_ok {
17827 let cause = runner.state.verification_blocking_cause().unwrap();
17828 assert!(cause.contains(&head));
17829 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17830 runner.state.event("land", "rebased successfully");
17832 let notice = crate::notices::run_ended(&runner.state).unwrap();
17833 assert!(!format!("{notice:?}").contains("rebased successfully"));
17834 assert!(runner.state.merge.is_none());
17835 }
17836 let before = std::fs::read_to_string(&phases).unwrap();
17837 runner.state.save().unwrap();
17838 runner.state = RunState::load(&runner.state.id).unwrap();
17839 runner.review_loop().await.unwrap();
17840 runner.gate().await.unwrap();
17841 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17842 assert_eq!(runner.state.verification_history.len(), 1);
17843 let history = &runner.state.verification_history[0];
17844 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17845 assert!(history.archived_at.is_some());
17846 assert_eq!(history.fixes_spent, Some(1));
17847 }
17848 }
17849
17850 #[cfg(unix)]
17851 #[tokio::test]
17852 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17853 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17854 let marker = tmp.path().join("gate-ran");
17855 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17856 runner.review_loop().await.unwrap();
17857 runner.gate().await.unwrap();
17858 assert_eq!(runner.state.status, RunStatus::Blocked);
17859 assert!(!marker.exists());
17860 assert!(runner.state.verification_history.is_empty());
17861 runner.state.gate_head = None;
17862 runner.gate().await.unwrap();
17863 assert!(marker.exists());
17864 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17865 assert_eq!(runner.state.verification_history.len(), 1);
17866 assert_eq!(runner.state.gate_fixes.len(), 1);
17867 }
17868
17869 #[cfg(unix)]
17870 #[tokio::test]
17871 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17872 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17873 runner.sync_to_base().await.unwrap();
17874 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17875 runner.state.reconcile_verification_head(&head);
17876 for outcomes in [
17878 Vec::new(),
17879 vec![CommandOutcome {
17880 command: "true".into(),
17881 code: None,
17882 output_tail: "cache unavailable".into(),
17883 duration_ms: 0,
17884 resource_blocked: true,
17885 }],
17886 ] {
17887 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17888 head: head.clone(),
17889 passed: false,
17890 outcomes,
17891 });
17892 runner.state.gate_ran = false;
17893 runner.state.gate.clear();
17894 runner.state.save().unwrap();
17895 runner.state = RunState::load(&runner.state.id).unwrap();
17896 runner.merge().await.unwrap();
17897 assert!(runner.state.merge.is_none());
17898 runner.review_loop().await.unwrap();
17899 runner.gate().await.unwrap();
17900 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17901 assert!(runner.state.gate_allows_progress());
17902 assert_eq!(runner.state.gate_fixes.len(), 1);
17903 assert!(runner.state.merge.is_none());
17904 }
17905 git::git(
17907 &runner.state.repo,
17908 &["commit", "--allow-empty", "-m", "external update"],
17909 )
17910 .await
17911 .unwrap();
17912 runner.merge().await.unwrap();
17913 assert!(runner.state.merge.is_none());
17914 assert_eq!(runner.state.status, RunStatus::Blocked);
17915 }
17916
17917 #[cfg(unix)]
17918 #[tokio::test]
17919 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17920 for waiting_on_gate in [false, true] {
17921 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17922 runner.sync_to_base().await.unwrap();
17923 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17924 runner.state.reconcile_verification_head(&head);
17925 if waiting_on_gate {
17926 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17927 head: head.clone(),
17928 passed: true,
17929 outcomes: Vec::new(),
17930 });
17931 }
17932 let cache = tmp.path().join("cache");
17933 runner.state.config.verify.e2e =
17934 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17935 runner.state.config.verify.gate = vec!["true".into()];
17936 runner.state.config.graph.timeout_resource_wait = 1;
17937 let owner =
17938 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17939 let guard =
17940 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17941 crate::cache::AcquireOutcome::Acquired(g) => g,
17942 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17943 };
17944 runner.review_loop().await.unwrap();
17945 runner.gate().await.unwrap();
17946 assert!(!runner.state.gate_ran);
17947 assert_eq!(runner.state.gate_fixes.len(), 1);
17948 assert!(runner.state.resource_block.is_some());
17949 runner.merge().await.unwrap();
17950 assert!(runner.state.merge.is_none());
17951 drop(guard);
17952 runner.state = RunState::load(&runner.state.id).unwrap();
17953 runner.state.resource_block = None;
17955 runner.review_loop().await.unwrap();
17956 runner.gate().await.unwrap();
17957 assert!(runner.state.gate_allows_progress());
17958 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17959 assert_eq!(runner.state.gate_fixes.len(), 1);
17960 }
17961 }
17962
17963 #[cfg(unix)]
17964 #[tokio::test]
17965 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17966 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17967 runner.sync_to_base().await.unwrap();
17968 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17969 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17970 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17972 runner.review_loop().await.unwrap();
17973 runner.gate().await.unwrap();
17974 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17975 assert_ne!(before, after);
17976 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17977 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17978 assert!(!runner.state.gate_allows_progress());
17979 runner.merge().await.unwrap();
17980 assert!(runner.state.merge.is_none());
17981 assert!(
17982 runner
17983 .state
17984 .verification_history
17985 .iter()
17986 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17987 );
17988
17989 runner.state.reviews[0].clean = false;
17991 runner.state.reviews[0].answered = 0;
17992 runner.state.config.verify.e2e = vec!["true".into()];
17993 runner.review_loop().await.unwrap();
17994 runner.gate().await.unwrap();
17995 runner.merge().await.unwrap();
17996 assert_eq!(runner.state.status, RunStatus::Blocked);
17997 assert!(runner.state.merge.is_none());
17998 assert!(
17999 runner
18000 .state
18001 .verification_blocking_cause()
18002 .unwrap()
18003 .contains("review blocked")
18004 );
18005 }
18006
18007 #[cfg(unix)]
18008 #[tokio::test]
18009 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18010 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18011 runner.state.gate_head = None;
18012 runner.state.config.verify.gate =
18013 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18014 runner.review_loop().await.unwrap();
18015 runner.gate().await.unwrap();
18016 assert_ne!(
18017 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18018 old
18019 );
18020 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18021 assert!(!runner.state.gate_allows_progress());
18022 assert_eq!(runner.state.status, RunStatus::Blocked);
18023 runner.merge().await.unwrap();
18024 assert!(runner.state.merge.is_none());
18025 }
18026
18027 #[cfg(unix)]
18028 #[tokio::test]
18029 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18030 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18031 let mut old_review = review_round(true, 0, 1, 1, false, true);
18032 old_review.head = head.clone();
18033 old_review.verified_head = Some(head.clone());
18034 runner.state.reviews = vec![old_review];
18035 runner.state.gate = vec![CommandOutcome {
18036 command: "old gate".into(),
18037 code: None,
18038 output_tail: "timed out after 2400s".into(),
18039 duration_ms: 2_400_000,
18040 resource_blocked: false,
18041 }];
18042 runner.state.gate_ran = true;
18043 runner.state.gate_head = Some(head.clone());
18044 runner.state.land_approval = Some(crate::run::LandApproval {
18045 question: "old-head-approval".into(),
18046 head: head.clone(),
18047 });
18048 runner.state.authorize_base_sync(&task).unwrap();
18049 runner.sync_to_base().await.unwrap();
18050 assert!(!runner.state.base_sync_blocked());
18051 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18052 assert_ne!(head, fresh);
18053 assert_eq!(runner.state.reviews.len(), 1);
18054 assert!(!runner.state.gate_ran);
18055 let saved = &runner.state.base_sync_recovery.authorizations[0];
18056 assert_eq!(saved.reviews[0].head, head);
18057 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18058 assert!(saved.gate_ran);
18059 assert!(!saved.gate[0].ok());
18060 runner.review_loop().await.unwrap();
18061 runner.gate().await.unwrap();
18062 assert_eq!(runner.state.reviews[0].head, head);
18063 assert_eq!(
18064 runner.state.reviews[0].verified_head.as_deref(),
18065 Some(head.as_str())
18066 );
18067 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18068 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18069 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18070 assert!(runner.state.gate_ran);
18071 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18072 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18073 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18074 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18075 assert_eq!(
18076 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18077 "retained change"
18078 );
18079 }
18080
18081 #[cfg(unix)]
18082 #[tokio::test]
18083 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18084 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18085 let queue = crate::queue::Queue::open();
18086 let mut owner = queue.get(&task).unwrap();
18087 owner.release_operator("test-resume", None);
18088 queue.put(&mut owner).unwrap();
18089 let generation = owner.resumption.as_ref().unwrap().generation;
18090 runner.state.task_instruction = Some(owner.instruction.clone());
18091 runner.state.driver_exited = true;
18092 runner.state.save().unwrap();
18093 let home = crate::run::home();
18094 let _claim = queue.claim(&task).unwrap();
18095 owner.resumption.as_mut().unwrap().at =
18096 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18097 queue.put(&mut owner).unwrap();
18098 assert!(
18099 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18100 .await
18101 .is_err(),
18102 "an approval from before the stop grants nothing"
18103 );
18104 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18105 queue.put(&mut owner).unwrap();
18106 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18107 .await
18108 .unwrap();
18109 assert!(
18110 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18111 .await
18112 .is_err()
18113 );
18114 runner.state = RunState::load(&runner.state.id).unwrap();
18115 assert_eq!(
18116 runner.state.base_sync_recovery.authorizations[0].generation,
18117 Some(generation)
18118 );
18119 runner.sync_to_base().await.unwrap();
18120 let repo = runner.state.repo.clone();
18121 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18122 git::git(&repo, &["checkout", "main"]).await.unwrap();
18123 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18124 git::git(&repo, &["add", "."]).await.unwrap();
18125 git::git(&repo, &["commit", "-m", "move base again"])
18126 .await
18127 .unwrap();
18128 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18129 git::git(&repo, &["checkout", "fix/retained"])
18130 .await
18131 .unwrap();
18132 runner.sync_to_base().await.unwrap();
18133 assert!(!runner.state.base_sync_blocked());
18134 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18135 assert_eq!(runner.state.base_episode_used(), 2);
18136 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18137 assert_ne!(fresh, head);
18138 assert!(!runner.state.base_recovery_pending());
18139 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18141 queue.put(&mut owner).unwrap();
18142 assert!(
18143 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18144 .await
18145 .is_err()
18146 );
18147 runner.state = RunState::load(&runner.state.id).unwrap();
18148 runner.sync_to_base().await.unwrap();
18149 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18150 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18151 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18152 }
18153
18154 #[cfg(unix)]
18155 #[tokio::test]
18156 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18157 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18158 runner.state.base_sync = None;
18159 runner.state.status = RunStatus::Blocked;
18160 let mut second = runner.state.candidates[0].clone();
18161 second.label = 'B';
18162 runner.state.candidates.push(second);
18163 assert!(runner.state.votes.is_empty());
18164 assert!(runner.state.deliberation.is_empty());
18165 runner.deliberate().await.unwrap();
18166 runner.vote().await.unwrap();
18167 assert_eq!(runner.state.status, RunStatus::Blocked);
18168 assert!(runner.state.votes.is_empty());
18169 assert!(runner.state.deliberation.is_empty());
18170 }
18171
18172 fn runner_at(status: RunStatus) -> Runner {
18175 let mut state = RunState::new(
18176 PathBuf::from("/nonexistent/repo"),
18177 "main".to_owned(),
18178 "deadbeef".to_owned(),
18179 "task".to_owned(),
18180 Config::default(),
18181 );
18182 state.config.disk.min_free_bytes = 0;
18184 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18187 state.id = format!(
18188 "20000101-000000-{:04x}",
18189 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18190 );
18191 state.status = status;
18192 Runner {
18193 state,
18194 roles: ResolvedRoles {
18195 implementers: Vec::new(),
18196 judges: Vec::new(),
18197 reviewers: Vec::new(),
18198 fixer: None,
18199 conductor: conductor(),
18200 implementer_roster: Vec::new(),
18201 judge_roster: Vec::new(),
18202 reviewer_roster: Vec::new(),
18203 },
18204 sem: Arc::new(Semaphore::new(1)),
18205 pause: Pause::new(),
18206 interrupt: Pause::new(),
18207 }
18208 }
18209
18210 #[tokio::test]
18211 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18212 crate::run::pin_test_home();
18213 let temp = tempfile::tempdir().unwrap();
18214 let root = temp.path().canonicalize().unwrap();
18215 let repo = root.join("repo");
18216 std::fs::create_dir(&repo).unwrap();
18217 init_repo(&repo);
18218 git::git(&repo, &["config", "core.autocrlf", "false"])
18219 .await
18220 .unwrap();
18221 let mut runner = runner_at(RunStatus::Blocked);
18222 runner.state.repo = repo.clone();
18223 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18224 runner.state.config.agents = vec![conductor()];
18225 runner.state.config.graph.selected_implementer = Some("conductor".into());
18226 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18227 let (shell, command, setup_output) = if cfg!(windows) {
18228 (
18229 vec!["cmd".into(), "/C".into()],
18230 "echo ready>seat-ready",
18231 "ready\r\n",
18232 )
18233 } else {
18234 (
18235 vec!["sh".into(), "-c".into()],
18236 "printf ready > seat-ready",
18237 "ready",
18238 )
18239 };
18240 runner.state.config.verify.shell = Some(shell);
18241 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18242 run: Some(command.into()),
18243 ..Default::default()
18244 }];
18245 runner.state.save().unwrap();
18246 let path = runner.state.worktree_root().join("judge-1");
18247 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18248 .await
18249 .unwrap();
18250 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18251 git::git(&path, &["add", "."]).await.unwrap();
18252 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18253 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18254 .await
18255 .unwrap();
18256 git::git(&path, &["checkout", "--detach", &head])
18257 .await
18258 .unwrap();
18259 let evidence = crate::fold_evidence::snapshot(
18260 &runner.state,
18261 &crate::run::home().canonicalize().unwrap(),
18262 &head,
18263 64 * 1024 * 1024,
18264 )
18265 .await
18266 .unwrap();
18267 let mut contents = BTreeMap::new();
18268 contents.insert(
18269 repo.join("logs/HEAD"),
18270 format!("{head} {head} fixture\n").into_bytes(),
18271 );
18272 let record = serde_json::json!({
18273 "version":1, "run":runner.state.id, "source":path, "head":head,
18274 "evidence":evidence, "admin":{}, "admin_contents":contents,
18275 "history":{head.clone():evidence}, "phase":"evicted"
18276 });
18277 std::fs::write(
18278 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18279 serde_json::to_vec(&record).unwrap(),
18280 )
18281 .unwrap();
18282 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18283 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18284 .await
18285 .unwrap();
18286 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18287 assert!(
18288 !git::git_raw(&repo, &["cat-file", "-e", &head])
18289 .await
18290 .unwrap()
18291 .ok()
18292 );
18293 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18294 let error = resumed.execute().await.unwrap_err();
18295 assert!(
18296 error
18297 .to_string()
18298 .contains("selected implementation is held"),
18299 "{error:#}"
18300 );
18301 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18302 assert_eq!(
18303 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18304 setup_output
18305 );
18306 assert_eq!(
18307 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18308 "unrelated seat"
18309 );
18310 let error = resumed.execute().await.unwrap_err();
18311 assert!(
18312 error
18313 .to_string()
18314 .contains("selected implementation is held"),
18315 "{error:#}"
18316 );
18317 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18318 git::git(&path, &["add", "."]).await.unwrap();
18319 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18320 let next = git::git(
18321 &path,
18322 &[
18323 "commit-tree",
18324 &next_tree,
18325 "-p",
18326 &head,
18327 "-m",
18328 "next review checkpoint",
18329 ],
18330 )
18331 .await
18332 .unwrap();
18333 git::git(&path, &["checkout", "--detach", &next])
18334 .await
18335 .unwrap();
18336 resumed.state.reviews.push(
18337 serde_json::from_value(serde_json::json!({
18338 "round":1, "head":next, "reviews":[]
18339 }))
18340 .unwrap(),
18341 );
18342 resumed.state.save().unwrap();
18343 let error = resumed.execute().await.unwrap_err();
18344 assert!(
18345 error
18346 .to_string()
18347 .contains("selected implementation is held"),
18348 "{error:#}"
18349 );
18350 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18351 }
18352
18353 #[tokio::test]
18354 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18355 crate::run::pin_test_home();
18356 let mut runner = runner_at(RunStatus::Gating);
18357 runner.state.save().unwrap();
18358 let id = runner.state.id.clone();
18359 let passes = runner.state.passes.len();
18360 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18361 let mut claiming = Box::pin(runner.claim_driver());
18362 std::future::poll_fn(|cx| {
18363 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18364 std::task::Poll::Ready(())
18365 })
18366 .await;
18367 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18368 drop(owner);
18369 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18370 .await
18371 .unwrap()
18372 .unwrap();
18373 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18374 assert!(execution.runner.state.resource_block.is_none());
18375 assert_eq!(
18376 execution.runner.state.liveness(false),
18377 crate::run::Liveness::Live
18378 );
18379 drop(execution);
18380 assert!(RunState::load(&id).unwrap().driver_exited);
18381 }
18382
18383 #[tokio::test]
18384 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18385 crate::run::pin_test_home();
18386 let mut runner = runner_at(RunStatus::Gating);
18387 runner.state.save().unwrap();
18388 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18391 assert!(runner.claim_driver().await.is_err());
18392 assert!(runner.state.driver_exited);
18393 assert_eq!(runner.state.driver_pid, None);
18394 let disk = RunState::load(&runner.state.id).unwrap();
18395 assert!(disk.driver_exited);
18396 assert_eq!(disk.driver_pid, None);
18397 assert_eq!(disk.status, RunStatus::Gating);
18398 }
18399
18400 #[tokio::test]
18401 async fn an_early_graph_error_releases_its_driver() {
18402 crate::run::pin_test_home();
18403 let mut runner = runner_at(RunStatus::Blocked);
18404 runner.state.config.graph.selected_implementer = Some("selected".into());
18405 runner.state.save().unwrap();
18406 assert!(runner.execute().await.is_err());
18407 let disk = RunState::load(&runner.state.id).unwrap();
18408 assert!(disk.driver_exited);
18409 assert_eq!(disk.driver_pid, None);
18410 assert_eq!(disk.status, RunStatus::Blocked);
18411 }
18412
18413 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18414 crate::run::MergeRepair {
18415 question: "repair-question".into(),
18416 head: "old-head".into(),
18417 pr: "https://github.com/o/r/pull/42".into(),
18418 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18419 gate: vec![outcome(Some(0), false)],
18420 gate_head: Some("old-head".into()),
18421 gate_fixes: Vec::new(),
18422 pre_gate: Vec::new(),
18423 phase,
18424 failure: None,
18425 }
18426 }
18427
18428 #[tokio::test]
18429 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18430 crate::run::pin_test_home();
18431 let mut runner = runner_at(RunStatus::Landing);
18432 runner.state.merge_repairs.push(merge_repair_request(
18433 crate::run::MergeRepairPhase::Requested,
18434 ));
18435 runner.state.save().unwrap();
18436 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18437 assert!(
18438 runner
18439 .execute()
18440 .await
18441 .unwrap_err()
18442 .to_string()
18443 .contains("already running")
18444 );
18445 assert_eq!(
18446 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18447 crate::run::MergeRepairPhase::Requested
18448 );
18449 drop(claim);
18450 let mut disk = runner.state.clone();
18451 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18452 disk.save().unwrap();
18453 assert!(
18454 runner
18455 .execute()
18456 .await
18457 .unwrap_err()
18458 .to_string()
18459 .contains("decision changed")
18460 );
18461 assert_eq!(
18462 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18463 crate::run::MergeRepairPhase::Complete
18464 );
18465 }
18466
18467 #[test]
18468 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18469 crate::run::pin_test_home();
18470 let mut runner = runner_at(RunStatus::Landing);
18471 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18472 runner.state.reviews = request.reviews.clone();
18473 runner.state.gate = request.gate.clone();
18474 runner.state.gate_ran = true;
18475 runner.state.land_approval = Some(crate::run::LandApproval {
18476 question: "old".into(),
18477 head: "old-head".into(),
18478 });
18479 runner.state.merge_repairs.push(request);
18480 runner.begin_merge_repair_review(0).unwrap();
18481 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18482 assert!(!runner.state.gate_status().ok());
18483 assert!(runner.state.land_approval.is_none());
18484 let disk = RunState::load(&runner.state.id).unwrap();
18485 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18486 assert!(disk.merge_repairs[0].gate[0].ok());
18487 assert_eq!(
18488 disk.merge_repairs[0].phase,
18489 crate::run::MergeRepairPhase::Reviewing
18490 );
18491 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18492 assert!(disk.reviews.is_empty());
18493 }
18494
18495 #[tokio::test]
18496 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18497 crate::run::pin_test_home();
18498 for phase in [
18499 crate::run::MergeRepairPhase::Fixing,
18500 crate::run::MergeRepairPhase::Requested,
18501 ] {
18502 let mut runner = runner_at(RunStatus::Landing);
18503 runner.state.config.graph.land_rounds = 0;
18504 runner.state.merge_repairs.push(merge_repair_request(phase));
18505 runner.state.save().unwrap();
18506 assert!(!runner.repair_merge_review().await.unwrap());
18507 assert_eq!(runner.state.status, RunStatus::Blocked);
18508 assert_eq!(
18509 runner.state.merge_repairs[0].phase,
18510 crate::run::MergeRepairPhase::Failed
18511 );
18512 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18513 assert!(
18514 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18515 "interrupted"
18516 } else {
18517 "budget exhausted"
18518 })
18519 );
18520 assert!(runner.state.merge.is_none());
18521 }
18522 }
18523
18524 #[test]
18528 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18529 crate::run::pin_test_home();
18530 let mut runner = runner_at(RunStatus::Implementing);
18531 let interrupt = Pause::new();
18532 runner.watch_interrupt(interrupt.clone());
18533
18534 interrupt.park_because("task a1b2 asked to run first");
18535
18536 assert!(runner.park_here().expect("park_here"));
18537 assert!(runner.state.parked);
18538 let last = runner.state.events.last().expect("a park event");
18539 assert_eq!(last.node, "park");
18540 assert!(
18541 last.message.contains("task a1b2 asked to run first"),
18542 "expected the interrupt reason in {:?}",
18543 last.message
18544 );
18545 }
18546
18547 #[test]
18550 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18551 crate::run::pin_test_home();
18552 let mut runner = runner_at(RunStatus::Implementing);
18553 runner.state.begin_pass();
18554 let shutdown = Pause::new();
18555 runner.on_pause(shutdown.clone());
18556 let upgrade = ParkCause::Upgrade {
18557 from: "0.121.0".to_owned(),
18558 to: Some("0.121.1".to_owned()),
18559 };
18560 shutdown.park_for(upgrade.clone());
18561 shutdown.park_for(ParkCause::Operator);
18562
18563 assert!(runner.park_here().expect("park_here"));
18564 assert_eq!(
18565 runner.state.passes[0].end,
18566 Some(crate::run::PassEnd::Parked { cause: upgrade })
18567 );
18568 let last = runner.state.events.last().expect("a park event");
18569 assert!(last.message.contains("0.121.0"), "{}", last.message);
18570 }
18571
18572 #[tokio::test]
18573 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18574 crate::run::pin_test_home();
18575 let mut runner = runner_at(RunStatus::Reviewing);
18576 runner.state.save().unwrap();
18577 let mut execution = runner.claim_driver().await.expect("first execution");
18578 assert_eq!(execution.runner.state.passes.len(), 1);
18579 execution
18580 .driver
18581 .finish(&mut execution.runner.state)
18582 .await
18583 .unwrap();
18584 }
18585
18586 #[tokio::test]
18587 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18588 crate::run::pin_test_home();
18589 let mut runner = runner_at(RunStatus::Reviewing);
18590 runner.state.config.agents = vec![conductor()];
18591 runner.state.config.graph.timeout_resource_wait = 0;
18592 runner.state.save().unwrap();
18593 let id = runner.state.id.clone();
18594 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18595 assert!(runner.claim_driver().await.is_err());
18596 drop(owner);
18597 drop(runner);
18598 let mut resumed = Runner::resume(&id).unwrap();
18599 assert_eq!(resumed.state.driver_claimed, Some(false));
18600 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18601 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18602 let mut execution = resumed
18603 .claim_driver()
18604 .await
18605 .expect("retry initial claim after reload");
18606 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18607 assert_eq!(execution.runner.state.passes.len(), 1);
18608 execution
18609 .driver
18610 .finish(&mut execution.runner.state)
18611 .await
18612 .unwrap();
18613 }
18614
18615 #[tokio::test]
18616 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18617 crate::run::pin_test_home();
18618 let mut runner = runner_at(RunStatus::Reviewing);
18619 runner.state.config.agents = vec![conductor()];
18620 runner.state.save().unwrap();
18621 let id = runner.state.id.clone();
18622 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18623 let mut resumed = Runner::resume(&id).unwrap();
18624 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18625 assert!(resumed.claim_driver().await.is_err());
18626 assert!(RunState::load(&id).unwrap().passes.is_empty());
18627 drop(lease);
18628 let mut execution = resumed.claim_driver().await.unwrap();
18629 execution
18630 .driver
18631 .finish(&mut execution.runner.state)
18632 .await
18633 .unwrap();
18634 }
18635
18636 #[tokio::test]
18637 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18638 crate::run::pin_test_home();
18639 let mut runner = runner_at(RunStatus::Reviewing);
18640 let state = &mut runner.state;
18641 state.config.agents = vec![conductor()];
18642 state.driver_claimed = None;
18643 state.driver_pid = Some(std::process::id());
18644 state.driver_started_at = None;
18645 state.begin_pass();
18646 state.save().unwrap();
18647 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18648 std::fs::write(
18649 state.dir().join("driver.json"),
18650 serde_json::to_vec(&serde_json::json!({
18651 "pid": state.driver_pid, "started_at": null,
18652 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18653 }))
18654 .unwrap(),
18655 )
18656 .unwrap();
18657 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18658 drop(lease);
18659 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18660 let mut resumed = Runner::resume(&state.id).unwrap();
18661 assert!(!resumed.state.driver_exited);
18662 let mut execution = resumed
18663 .claim_driver()
18664 .await
18665 .expect("lease proves previous execution ended");
18666 assert_eq!(execution.runner.state.passes.len(), 2);
18667 execution
18668 .driver
18669 .finish(&mut execution.runner.state)
18670 .await
18671 .unwrap();
18672 }
18673
18674 #[tokio::test]
18675 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18676 crate::run::pin_test_home();
18677 let mut runner = runner_at(RunStatus::Reviewing);
18678 let mut json = serde_json::to_value(&runner.state).unwrap();
18679 json.as_object_mut().unwrap().remove("driver_claimed");
18680 runner.state = serde_json::from_value(json).unwrap();
18681 assert_eq!(runner.state.driver_claimed, None);
18682 runner.state.save().unwrap();
18683 let mut execution = runner
18684 .claim_driver()
18685 .await
18686 .expect("the execution lease, not a missing marker, excludes a second driver");
18687 assert_eq!(execution.runner.state.passes.len(), 1);
18688 execution
18689 .driver
18690 .finish(&mut execution.runner.state)
18691 .await
18692 .unwrap();
18693 }
18694
18695 #[tokio::test]
18696 async fn driver_admission_refuses_a_proven_live_driver() {
18697 crate::run::pin_test_home();
18698 let mut runner = runner_at(RunStatus::Reviewing);
18699 let pid = std::process::id();
18700 runner.state.driver_pid = Some(pid);
18701 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18702 assert!(runner.state.driver_started_at.is_some());
18703 runner.state.save().unwrap();
18704 let error = runner
18705 .claim_driver()
18706 .await
18707 .err()
18708 .expect("a live driver must refuse");
18709 assert!(error.to_string().contains("live driver"), "{error:#}");
18710 let saved = RunState::load(&runner.state.id).unwrap();
18711 assert_eq!(saved.driver_pid, Some(pid));
18712 assert_eq!(saved.status, RunStatus::Reviewing);
18713 assert!(saved.passes.is_empty());
18714 }
18715
18716 #[test]
18717 fn parking_resource_halt_persists_the_pass_once() {
18718 crate::run::pin_test_home();
18719 let mut runner = runner_at(RunStatus::Reviewing);
18720 runner.state.begin_pass();
18721 runner.state.resource_block = Some(crate::run::ResourceBlock {
18722 node: "fix".into(),
18723 kind: "shared build cache".into(),
18724 detail: "cancelled".into(),
18725 at: Timestamp::now(),
18726 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18727 });
18728 let cause = ParkCause::Upgrade {
18729 from: "old".into(),
18730 to: Some("new".into()),
18731 };
18732 runner.pause.park_for(cause.clone());
18733 runner.stop_for_resource().unwrap();
18736 assert!(runner.resource_halt().unwrap());
18737 assert!(runner.park_here().unwrap());
18738 let saved = RunState::load(&runner.state.id).unwrap();
18739 assert!(saved.parked);
18740 assert_eq!(saved.status, RunStatus::Reviewing);
18741 assert!(saved.passes[0].ended_at.is_some());
18742 assert_eq!(
18743 saved.passes[0].end,
18744 Some(crate::run::PassEnd::Parked {
18745 cause: cause.clone()
18746 })
18747 );
18748 assert_eq!(
18749 saved.resource_block.unwrap().cause,
18750 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18751 );
18752 assert_eq!(
18753 saved
18754 .events
18755 .iter()
18756 .filter(|event| event.node == "park")
18757 .count(),
18758 1
18759 );
18760 }
18761
18762 #[test]
18770 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18771 crate::run::pin_test_home();
18772 let mut runner = runner_at(RunStatus::Implementing);
18773 let shutdown = Pause::new();
18774 runner.on_pause(shutdown.clone());
18775 let interrupt = Pause::new();
18776 runner.watch_interrupt(interrupt.clone());
18777
18778 assert!(!runner.park_here().expect("park_here"));
18780 assert!(!runner.state.parked);
18781
18782 interrupt.park_because("test");
18784 assert!(!shutdown.parked());
18785 assert!(runner.park_here().expect("park_here"));
18786 }
18787
18788 #[tokio::test]
18802 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18803 crate::run::pin_test_home();
18804 let mut runner = runner_at(RunStatus::Implementing);
18805 let interrupt = Pause::new();
18806 runner.watch_interrupt(interrupt.clone());
18807
18808 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18809 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18810
18811 let node = async move {
18815 started_tx.send(()).expect("send started");
18816 finish_rx.await.expect("recv finish");
18817 "node finished"
18818 };
18819
18820 let interrupter = async move {
18821 started_rx.await.expect("recv started");
18822 interrupt.park_because("higher-priority task waiting");
18824 tokio::task::yield_now().await;
18828 finish_tx.send(()).expect("send finish");
18829 };
18830
18831 let (node_result, ()) = tokio::join!(node, interrupter);
18832 assert_eq!(
18833 node_result, "node finished",
18834 "the in-flight call ran to completion"
18835 );
18836
18837 assert!(runner.park_here().expect("park_here"));
18840 assert!(runner.state.parked);
18841 }
18842
18843 #[test]
18849 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18850 crate::run::pin_test_home();
18851 let mut runner = runner_at(RunStatus::Judging);
18852 runner.state.config.agents = vec![conductor()];
18856 runner.state.candidates = vec![Candidate {
18857 index: 0,
18858 label: 'A',
18859 agent: "alpha".to_owned(),
18860 branch: "magi/x/A".to_owned(),
18861 worktree: PathBuf::from("/nonexistent/worktree"),
18862 summary: "did the thing".to_owned(),
18863 stat: "1 file changed".to_owned(),
18864 files: 1,
18865 commits: 1,
18866 empty: false,
18867 failed: None,
18868 verified_noop: None,
18869 duration_ms: 1234,
18870 folded: false,
18871 }];
18872 let run_id = runner.state.id.clone();
18873
18874 let interrupt = Pause::new();
18875 runner.watch_interrupt(interrupt.clone());
18876 interrupt.park_because("task c3d4 asked to run first");
18877 assert!(runner.park_here().expect("park_here"));
18878
18879 let resumed = Runner::resume(&run_id).expect("resume");
18880 assert_eq!(resumed.state.candidates.len(), 1);
18881 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18882 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18883 assert_eq!(resumed.state.status, runner.state.status);
18884 assert!(
18885 resumed.state.parked,
18886 "still parked until `execute` actually walks the graph again"
18887 );
18888 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18889 }
18890
18891 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18893 let mut q = ask::Question::new(
18894 run.to_owned(),
18895 "implement".to_owned(),
18896 "impl-A".to_owned(),
18897 "Which storage backend should the cache use?".to_owned(),
18898 String::new(),
18899 vec!["SQLite".to_owned(), "Redis".to_owned()],
18900 );
18901 store.put(&mut q).unwrap();
18902 q
18903 }
18904
18905 #[test]
18906 fn a_failed_runs_open_question_is_abandoned() {
18907 ask_test_home();
18908 let store = ask::Questions::open();
18909 let mut runner = runner_at(RunStatus::Failed);
18910 let run = runner.state.id.clone();
18911 let q = ask_open_question(&store, &run);
18912
18913 runner.settle_questions();
18914
18915 let back = store.get(&q.id).unwrap();
18916 assert!(
18917 !back.status.open(),
18918 "the seat that asked died with the run; nobody is left to read an answer"
18919 );
18920 assert!(
18921 back.detail.contains(&run) && back.detail.contains("failed"),
18922 "the reason names what the run became, not just that it is gone: {}",
18923 back.detail
18924 );
18925 }
18926
18927 #[test]
18928 fn a_merged_runs_open_question_is_abandoned_too() {
18929 ask_test_home();
18930 let store = ask::Questions::open();
18931 for status in [RunStatus::Merged, RunStatus::Ready] {
18934 let mut runner = runner_at(status);
18935 let run = runner.state.id.clone();
18936 let q = ask_open_question(&store, &run);
18937
18938 runner.settle_questions();
18939
18940 let back = store.get(&q.id).unwrap();
18941 assert!(
18942 !back.status.open(),
18943 "{status:?} run's question must not outlive the run"
18944 );
18945 }
18946 }
18947
18948 #[test]
18949 fn a_still_resumable_runs_open_question_is_left_alone() {
18950 ask_test_home();
18951 let store = ask::Questions::open();
18952 for status in [RunStatus::Blocked, RunStatus::Stalled] {
18958 let mut runner = runner_at(status);
18959 let run = runner.state.id.clone();
18960 let q = ask_open_question(&store, &run);
18961
18962 runner.settle_questions();
18963
18964 let back = store.get(&q.id).unwrap();
18965 assert!(
18966 back.status.open(),
18967 "{status:?} is still alive; the question must still be waiting"
18968 );
18969 }
18970 }
18971
18972 #[test]
18973 fn settle_questions_never_touches_an_already_answered_question() {
18974 ask_test_home();
18975 let store = ask::Questions::open();
18976 let mut runner = runner_at(RunStatus::Failed);
18977 let run = runner.state.id.clone();
18978 let mut q = ask_open_question(&store, &run);
18979 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18980 .unwrap();
18981 store.put(&mut q).unwrap();
18982
18983 runner.settle_questions();
18988 runner.settle_questions();
18989
18990 let back = store.get(&q.id).unwrap();
18991 assert_eq!(
18992 back.status,
18993 ask::QuestionStatus::Answered,
18994 "a real answer is a decision on record, never overwritten by a sweep"
18995 );
18996 }
18997
18998 #[tokio::test]
19009 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19010 crate::run::pin_test_home();
19011 let tmp = tempfile::tempdir().expect("tempdir");
19012 let repo = tmp.path().join("repo");
19013 std::fs::create_dir_all(&repo).unwrap();
19014 init_repo(&repo);
19015
19016 let mut config = Config::default();
19017 config.graph.worktree_root = Some(tmp.path().join("wt"));
19018
19019 let mut state = RunState::new(
19020 repo.clone(),
19021 "main".to_owned(),
19022 "deadbeef".to_owned(),
19023 "task".to_owned(),
19024 config,
19025 );
19026 let root = state.worktree_root();
19027 let wt_a = root.join("cand-A");
19028 let wt_b = root.join("cand-B");
19029 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19030 .await
19031 .expect("worktree A");
19032 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19033 .await
19034 .expect("worktree B");
19035
19036 state.candidates = vec![
19037 Candidate {
19038 index: 0,
19039 label: 'A',
19040 agent: "alpha".to_owned(),
19041 branch: "magi/x/A".to_owned(),
19042 worktree: wt_a.clone(),
19043 summary: String::new(),
19044 stat: String::new(),
19045 files: 0,
19046 commits: 0,
19047 empty: false,
19048 failed: None,
19049 verified_noop: None,
19050 duration_ms: 0,
19051 folded: false,
19052 },
19053 Candidate {
19054 index: 1,
19055 label: 'B',
19056 agent: "beta".to_owned(),
19057 branch: "magi/x/B".to_owned(),
19058 worktree: wt_b.clone(),
19059 summary: String::new(),
19060 stat: String::new(),
19061 files: 0,
19062 commits: 0,
19063 empty: false,
19064 failed: None,
19065 verified_noop: None,
19066 duration_ms: 0,
19067 folded: false,
19068 },
19069 ];
19070 state.tally = Some(Tally {
19071 first_choice: BTreeMap::from([('A', 1)]),
19072 borda: BTreeMap::new(),
19073 winner: 'A',
19074 rankings: 1,
19075 unanimous_initial: true,
19076 deliberated: false,
19077 changed_votes: 0,
19078 unanimous_final: true,
19079 tie_break: None,
19080 judges: 1,
19081 present: 1,
19082 quorum: 1,
19083 met_quorum: true,
19084 uncontested: None,
19085 });
19086 state.status = RunStatus::Ready;
19087
19088 fold_run(&mut state, false, &crate::run::home())
19089 .await
19090 .expect("fold_run");
19091
19092 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19093 assert!(
19094 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19095 "the unmerged winner's branch survives"
19096 );
19097 assert!(
19098 !state.candidates[0].folded,
19099 "the winner is not marked folded"
19100 );
19101
19102 assert!(!wt_b.exists(), "the loser's worktree is removed");
19103 assert!(
19104 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19105 "the loser's branch is removed"
19106 );
19107 assert!(state.candidates[1].folded, "the loser is marked folded");
19108 }
19109
19110 #[tokio::test]
19111 #[cfg(unix)]
19112 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19113 use std::os::fd::AsRawFd as _;
19114 use std::time::Duration;
19115
19116 let tmp = tempfile::tempdir().unwrap();
19117 let repo = tmp.path().join("repo");
19118 std::fs::create_dir_all(&repo).unwrap();
19119 init_repo(&repo);
19120 let common = repo.join(".git");
19121 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19122 let home = tmp.path().join("home");
19123 let mut config = Config::default();
19124 config.coordination = crate::config::test_coordination();
19125 config.graph.worktree_root = Some(tmp.path().join("wt"));
19126 config.graph.timeout_resource_wait = 1;
19127 let mut state = RunState::new(
19128 repo.clone(),
19129 "main".into(),
19130 "deadbeef".into(),
19131 "task".into(),
19132 config,
19133 );
19134 let root = state.worktree_root();
19135 let candidate = root.join("cand-A");
19136 let reviewer = root.join("review-1");
19137 let branch = state.branch_for('A');
19138 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19139 .await
19140 .unwrap();
19141 git::worktree_add_detached(&repo, &reviewer, "main")
19142 .await
19143 .unwrap();
19144 git::acquire_worktree_config(&repo).await.unwrap();
19145 state.enabled_worktree_config = true;
19146 state.status = RunStatus::Ready;
19147 state.candidates.push(Candidate {
19148 index: 0,
19149 label: 'A',
19150 agent: "implement".into(),
19151 branch: branch.clone(),
19152 worktree: candidate.clone(),
19153 summary: String::new(),
19154 stat: String::new(),
19155 files: 0,
19156 commits: 0,
19157 empty: false,
19158 failed: None,
19159 verified_noop: None,
19160 duration_ms: 0,
19161 folded: false,
19162 });
19163 state.save_under(&home).unwrap();
19164 let config_before = std::fs::read(common.join("config")).unwrap();
19165 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19166 .await
19167 .unwrap();
19168 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19171 assert_eq!(
19172 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19173 0
19174 );
19175 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19176 assert!(
19177 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19178 .await
19179 .is_err()
19180 );
19181 assert!(candidate.exists());
19182 assert!(reviewer.exists());
19183 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19184 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19185 assert_eq!(
19186 git::git(&repo, &["worktree", "list", "--porcelain"])
19187 .await
19188 .unwrap(),
19189 worktrees_before
19190 );
19191 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19192 .await
19193 .unwrap()
19194 .unwrap_err();
19195 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19196 drop(fold);
19197 assert!(candidate.exists() && reviewer.exists());
19198 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19199 assert_eq!(
19200 state.resource_block.as_ref().unwrap().kind,
19201 "repository coordination"
19202 );
19203 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19204 drop(external);
19205 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19206 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19207 .await
19208 .unwrap()
19209 .unwrap();
19210 drop(fold);
19211 assert!(removed.contains(&branch));
19212 assert!(!root.exists());
19213 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19214 assert!(state.candidates[0].folded);
19215 assert!(!state.enabled_worktree_config);
19216 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19217 let enabled = git::git_raw(
19218 &repo,
19219 &["config", "--local", "--get", "extensions.worktreeConfig"],
19220 )
19221 .await
19222 .unwrap();
19223 assert!(!enabled.ok(), "the last config reference was released");
19224 }
19225
19226 #[tokio::test]
19229 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19230 let tmp = tempfile::tempdir().expect("tempdir");
19231 let repo = tmp.path().join("repo");
19232 std::fs::create_dir_all(&repo).unwrap();
19233 init_repo(&repo);
19234 let home = tmp.path().join("home");
19235
19236 let mut config = Config::default();
19237 config.graph.worktree_root = Some(tmp.path().join("wt"));
19238 let mut state = RunState::new(
19239 repo.clone(),
19240 "main".to_owned(),
19241 "deadbeef".to_owned(),
19242 "task".to_owned(),
19243 config,
19244 );
19245 git::git(&repo, &["branch", "magi/x/A", "main"])
19247 .await
19248 .expect("branch");
19249 state.candidates = vec![Candidate {
19250 index: 0,
19251 label: 'A',
19252 agent: "alpha".to_owned(),
19253 branch: "magi/x/A".to_owned(),
19254 worktree: state.worktree_root().join("cand-A"),
19255 summary: String::new(),
19256 stat: String::new(),
19257 files: 0,
19258 commits: 0,
19259 empty: false,
19260 failed: None,
19261 verified_noop: None,
19262 duration_ms: 0,
19263 folded: true,
19264 }];
19265 state.released_to = Some("20260901-000000-new1".to_owned());
19266 state.released_branches = vec!["magi/x/A".to_owned()];
19267
19268 fold_run(&mut state, true, &home).await.expect("fold_run");
19269 state.status = RunStatus::Merged;
19270 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19271 state.merge = Some(crate::run::MergeOutcome {
19272 mode: MergeMode::Pr,
19273 ok: true,
19274 detail: "confirmed merged".into(),
19275 empty: false,
19276 });
19277 fold_run(&mut state, true, &home)
19278 .await
19279 .expect("merged fold");
19280
19281 assert!(
19282 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19283 "the handed-over branch survives a fold"
19284 );
19285 }
19286
19287 #[tokio::test]
19291 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19292 let tmp = tempfile::tempdir().expect("tempdir");
19293 let repo = tmp.path().join("repo");
19294 std::fs::create_dir_all(&repo).unwrap();
19295 init_repo(&repo);
19296 let run = |args: &[&str]| {
19297 let out = std::process::Command::new("git")
19298 .quiet()
19299 .args(args)
19300 .current_dir(&repo)
19301 .output()
19302 .expect("spawn git");
19303 assert!(out.status.success(), "git {args:?}");
19304 };
19305 run(&["branch", "magi/x/A"]);
19306 run(&["checkout", "-q", "-b", "magi/x/B"]);
19307 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19308 run(&["add", "-A"]);
19309 run(&["commit", "-q", "-m", "work"]);
19310 run(&["checkout", "-q", "main"]);
19311
19312 let probe = RunState::new(
19316 repo.clone(),
19317 "main".to_owned(),
19318 "deadbeef".to_owned(),
19319 "task".to_owned(),
19320 Config::default(),
19321 );
19322 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19323 let bare = tmp.path().join("origin.git");
19324 let out = std::process::Command::new("git")
19325 .quiet()
19326 .args(["init", "-q", "--bare"])
19327 .arg(&bare)
19328 .output()
19329 .expect("spawn git");
19330 assert!(out.status.success(), "git init --bare");
19331 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19332 run(&["push", "-q", "origin", "main"]);
19333
19334 let mut state = RunState::new(
19335 repo.clone(),
19336 "main".to_owned(),
19337 "deadbeef".to_owned(),
19338 "task".to_owned(),
19339 Config::default(),
19340 );
19341 state.seeds = vec![refs::Seed {
19342 token: "magi/27b2/A".to_owned(),
19343 kind: refs::SeedKind::Unresolved,
19344 sha: String::new(),
19345 branch: true,
19346 detail: "no branch or commit named magi/27b2/A".to_owned(),
19347 }];
19348
19349 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19350 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19351 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19352 let detail = empty_candidate_detail(&state, "main");
19353 assert!(detail.starts_with("empty candidate"), "{detail}");
19354 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19355 assert!(detail.contains("--start-from"), "{detail}");
19356 }
19357
19358 #[tokio::test]
19367 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19368 let tmp = tempfile::tempdir().expect("tempdir");
19369 let repo = tmp.path().join("repo");
19370 std::fs::create_dir_all(&repo).unwrap();
19371 init_repo(&repo);
19372
19373 let mut config = Config::default();
19374 config.merge.mode = MergeMode::Local;
19375
19376 let mut state = RunState::new(
19377 repo.clone(),
19378 "main".to_owned(),
19379 "deadbeef".to_owned(),
19380 "task".to_owned(),
19381 config,
19382 );
19383 state.candidates = vec![Candidate {
19384 index: 0,
19385 label: 'A',
19386 agent: "alpha".to_owned(),
19387 branch: "does-not-exist".to_owned(),
19388 worktree: repo.clone(),
19389 summary: String::new(),
19390 stat: String::new(),
19391 files: 0,
19392 commits: 0,
19393 empty: false,
19394 failed: None,
19395 verified_noop: None,
19396 duration_ms: 0,
19397 folded: false,
19398 }];
19399 state.tally = Some(Tally {
19400 first_choice: BTreeMap::from([('A', 1)]),
19401 borda: BTreeMap::new(),
19402 winner: 'A',
19403 rankings: 1,
19404 unanimous_initial: true,
19405 deliberated: false,
19406 changed_votes: 0,
19407 unanimous_final: true,
19408 tie_break: None,
19409 judges: 0,
19410 present: 0,
19411 quorum: 0,
19412 met_quorum: true,
19413 uncontested: Some("only candidate A produced a change".to_owned()),
19414 });
19415 state.reviews = vec![ReviewRound {
19416 round: 1,
19417 head: "deadbeef".to_owned(),
19418 verified_head: None,
19419 verified_at: None,
19420 reviews: Vec::new(),
19421 e2e: Vec::new(),
19422 fix: None,
19423 blocking: 0,
19424 answered: 0,
19425 expected: 0,
19426 clean: true,
19427 verify_retried: false,
19428 e2e_deferred: false,
19429 e2e_defer_reason: None,
19430 progressed: false,
19431 vote_split: false,
19432 reconsideration: Vec::new(),
19433 verdict: None,
19434 }];
19435 state.gate = vec![CommandOutcome {
19436 command: "test".to_owned(),
19437 code: Some(0),
19438 output_tail: String::new(),
19439 duration_ms: 0,
19440 resource_blocked: false,
19441 }];
19442 state.gate_ran = true;
19443 state.status = RunStatus::Ready;
19448 state.merge = Some(MergeOutcome {
19449 mode: MergeMode::Local,
19450 ok: false,
19451 detail: "already concluded".to_owned(),
19452 empty: false,
19453 });
19454
19455 let mut runner = Runner {
19456 state,
19457 roles: ResolvedRoles {
19458 implementers: Vec::new(),
19459 judges: Vec::new(),
19460 reviewers: Vec::new(),
19461 fixer: None,
19462 conductor: conductor(),
19463 implementer_roster: Vec::new(),
19464 judge_roster: Vec::new(),
19465 reviewer_roster: Vec::new(),
19466 },
19467 sem: Arc::new(Semaphore::new(1)),
19468 pause: Pause::new(),
19469 interrupt: Pause::new(),
19470 };
19471
19472 runner.merge().await.expect("merge");
19473
19474 assert_eq!(
19475 runner.state.status,
19476 RunStatus::Ready,
19477 "a concluded run's status must not change on reentry"
19478 );
19479 assert_eq!(
19480 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19481 Some("already concluded"),
19482 "merge must not run again once the node already recorded an outcome"
19483 );
19484 }
19485
19486 #[tokio::test]
19495 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19496 let tmp = tempfile::tempdir().expect("tempdir");
19497 let repo = tmp.path().join("repo");
19498 std::fs::create_dir_all(&repo).unwrap();
19499 init_repo(&repo);
19500
19501 let mut config = Config::default();
19502 config.merge.mode = MergeMode::Local;
19503
19504 let mut state = RunState::new(
19505 repo.clone(),
19506 "main".to_owned(),
19507 "deadbeef".to_owned(),
19508 "task".to_owned(),
19509 config,
19510 );
19511 state.candidates = vec![Candidate {
19512 index: 0,
19513 label: 'A',
19514 agent: "alpha".to_owned(),
19515 branch: "does-not-exist".to_owned(),
19516 worktree: repo.clone(),
19517 summary: String::new(),
19518 stat: String::new(),
19519 files: 0,
19520 commits: 0,
19521 empty: false,
19522 failed: None,
19523 verified_noop: None,
19524 duration_ms: 0,
19525 folded: false,
19526 }];
19527 state.tally = Some(Tally {
19528 first_choice: BTreeMap::from([('A', 1)]),
19529 borda: BTreeMap::new(),
19530 winner: 'A',
19531 rankings: 1,
19532 unanimous_initial: true,
19533 deliberated: false,
19534 changed_votes: 0,
19535 unanimous_final: true,
19536 tie_break: None,
19537 judges: 0,
19538 present: 0,
19539 quorum: 0,
19540 met_quorum: true,
19541 uncontested: Some("only candidate A produced a change".to_owned()),
19542 });
19543 state.reviews = vec![ReviewRound {
19544 round: 1,
19545 head: "deadbeef".to_owned(),
19546 verified_head: None,
19547 verified_at: None,
19548 reviews: Vec::new(),
19549 e2e: Vec::new(),
19550 fix: None,
19551 blocking: 0,
19552 answered: 0,
19553 expected: 0,
19554 clean: true,
19555 verify_retried: false,
19556 e2e_deferred: false,
19557 e2e_defer_reason: None,
19558 progressed: false,
19559 vote_split: false,
19560 reconsideration: Vec::new(),
19561 verdict: None,
19562 }];
19563 state.gate = Vec::new();
19565 state.gate_ran = false;
19566 state.status = RunStatus::Gating;
19567
19568 let mut runner = Runner {
19569 state,
19570 roles: ResolvedRoles {
19571 implementers: Vec::new(),
19572 judges: Vec::new(),
19573 reviewers: Vec::new(),
19574 fixer: None,
19575 conductor: conductor(),
19576 implementer_roster: Vec::new(),
19577 judge_roster: Vec::new(),
19578 reviewer_roster: Vec::new(),
19579 },
19580 sem: Arc::new(Semaphore::new(1)),
19581 pause: Pause::new(),
19582 interrupt: Pause::new(),
19583 };
19584
19585 runner.merge().await.expect("merge");
19586
19587 assert!(
19588 runner.state.merge.is_none(),
19589 "an empty gate must never be read as a passing one: {:?}",
19590 runner.state.merge
19591 );
19592 }
19593
19594 #[tokio::test]
19601 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19602 let tmp = tempfile::tempdir().expect("tempdir");
19603 let repo = tmp.path().join("repo");
19604 std::fs::create_dir_all(&repo).unwrap();
19605 init_repo(&repo);
19606
19607 let config = Config::default();
19609
19610 let mut state = RunState::new(
19611 repo.clone(),
19612 "main".to_owned(),
19613 "deadbeef".to_owned(),
19614 "task".to_owned(),
19615 config,
19616 );
19617 state.candidates = vec![Candidate {
19618 index: 0,
19619 label: 'A',
19620 agent: "alpha".to_owned(),
19621 branch: "does-not-exist".to_owned(),
19622 worktree: repo.clone(),
19623 summary: String::new(),
19624 stat: String::new(),
19625 files: 0,
19626 commits: 0,
19627 empty: false,
19628 failed: None,
19629 verified_noop: None,
19630 duration_ms: 0,
19631 folded: false,
19632 }];
19633 state.tally = Some(Tally {
19634 first_choice: BTreeMap::from([('A', 1)]),
19635 borda: BTreeMap::new(),
19636 winner: 'A',
19637 rankings: 1,
19638 unanimous_initial: true,
19639 deliberated: false,
19640 changed_votes: 0,
19641 unanimous_final: true,
19642 tie_break: None,
19643 judges: 0,
19644 present: 0,
19645 quorum: 0,
19646 met_quorum: true,
19647 uncontested: Some("only candidate A produced a change".to_owned()),
19648 });
19649 state.reviews = vec![ReviewRound {
19650 round: 1,
19651 head: "deadbeef".to_owned(),
19652 verified_head: None,
19653 verified_at: None,
19654 reviews: Vec::new(),
19655 e2e: Vec::new(),
19656 fix: None,
19657 blocking: 0,
19658 answered: 0,
19659 expected: 0,
19660 clean: true,
19661 verify_retried: false,
19662 e2e_deferred: false,
19663 e2e_defer_reason: None,
19664 progressed: false,
19665 vote_split: false,
19666 reconsideration: Vec::new(),
19667 verdict: None,
19668 }];
19669
19670 let mut runner = Runner {
19671 state,
19672 roles: ResolvedRoles {
19673 implementers: Vec::new(),
19674 judges: Vec::new(),
19675 reviewers: Vec::new(),
19676 fixer: None,
19677 conductor: conductor(),
19678 implementer_roster: Vec::new(),
19679 judge_roster: Vec::new(),
19680 reviewer_roster: Vec::new(),
19681 },
19682 sem: Arc::new(Semaphore::new(1)),
19683 pause: Pause::new(),
19684 interrupt: Pause::new(),
19685 };
19686
19687 runner.gate().await.expect("gate");
19688 assert!(
19689 runner.state.gate_ran,
19690 "zero configured commands is still a real attempt, not an unrun gate"
19691 );
19692 assert!(runner.state.gate.is_empty());
19693 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19694 assert_ne!(
19695 runner.state.status,
19696 RunStatus::Blocked,
19697 "a gate with nothing to check must not read as failed"
19698 );
19699
19700 runner.merge().await.expect("merge");
19701 assert_eq!(
19702 runner.state.status,
19703 RunStatus::Ready,
19704 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19705 );
19706 }
19707
19708 #[tokio::test]
19719 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19720 crate::run::pin_test_home();
19721 let home = crate::run::home();
19722
19723 let tmp = tempfile::tempdir().expect("tempdir");
19724 let repo = tmp.path().join("repo");
19725 std::fs::create_dir_all(&repo).unwrap();
19726 init_repo(&repo);
19727 let cache_dir = tmp.path().join("target");
19730
19731 let mut config = Config::default();
19732 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19733 config.graph.timeout_verify = Some(2);
19737 config.graph.timeout_resource_wait = 0;
19738
19739 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19740 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19741 .expect("no io error acquiring directly")
19742 {
19743 crate::cache::AcquireOutcome::Acquired(g) => g,
19744 crate::cache::AcquireOutcome::Busy(b) => {
19745 panic!("expected the direct acquire to win the lease first: {b:?}")
19746 }
19747 };
19748
19749 let mut state = RunState::new(
19750 repo.clone(),
19751 "main".to_owned(),
19752 "deadbeef".to_owned(),
19753 "task".to_owned(),
19754 config,
19755 );
19756 state.candidates = vec![Candidate {
19757 index: 0,
19758 label: 'A',
19759 agent: "alpha".to_owned(),
19760 branch: "does-not-exist".to_owned(),
19761 worktree: repo.clone(),
19762 summary: String::new(),
19763 stat: String::new(),
19764 files: 0,
19765 commits: 0,
19766 empty: false,
19767 failed: None,
19768 verified_noop: None,
19769 duration_ms: 0,
19770 folded: false,
19771 }];
19772 state.tally = Some(Tally {
19773 first_choice: BTreeMap::from([('A', 1)]),
19774 borda: BTreeMap::new(),
19775 winner: 'A',
19776 rankings: 1,
19777 unanimous_initial: true,
19778 deliberated: false,
19779 changed_votes: 0,
19780 unanimous_final: true,
19781 tie_break: None,
19782 judges: 0,
19783 present: 0,
19784 quorum: 0,
19785 met_quorum: true,
19786 uncontested: Some("only candidate A produced a change".to_owned()),
19787 });
19788 state.reviews = vec![ReviewRound {
19789 round: 1,
19790 head: "deadbeef".to_owned(),
19791 verified_head: None,
19792 verified_at: None,
19793 reviews: Vec::new(),
19794 e2e: Vec::new(),
19795 fix: None,
19796 blocking: 0,
19797 answered: 0,
19798 expected: 0,
19799 clean: true,
19800 verify_retried: false,
19801 e2e_deferred: false,
19802 e2e_defer_reason: None,
19803 progressed: false,
19804 vote_split: false,
19805 reconsideration: Vec::new(),
19806 verdict: None,
19807 }];
19808
19809 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19810 state.reviews[0].e2e = vec![CommandOutcome {
19811 command: state.config.verify.e2e[0].clone(),
19812 code: Some(0),
19813 output_tail: String::new(),
19814 duration_ms: 0,
19815 resource_blocked: false,
19816 }];
19817 let mut runner = Runner {
19818 state,
19819 roles: ResolvedRoles {
19820 implementers: Vec::new(),
19821 judges: Vec::new(),
19822 reviewers: Vec::new(),
19823 fixer: None,
19824 conductor: conductor(),
19825 implementer_roster: Vec::new(),
19826 judge_roster: Vec::new(),
19827 reviewer_roster: Vec::new(),
19828 },
19829 sem: Arc::new(Semaphore::new(1)),
19830 pause: Pause::new(),
19831 interrupt: Pause::new(),
19832 };
19833
19834 runner.gate().await.expect("gate");
19835 assert!(runner.state.resource_block.is_none());
19836 assert!(
19837 !runner
19838 .state
19839 .events
19840 .iter()
19841 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19842 "a gate with nothing to run must never request the held cache lease"
19843 );
19844 assert!(
19845 runner.state.gate_ran,
19846 "zero commands is still a real, immediate attempt"
19847 );
19848 assert!(runner.state.gate.is_empty());
19849 assert_ne!(
19850 runner.state.status,
19851 RunStatus::Blocked,
19852 "must not read as resource-blocked on a lease it never asked for"
19853 );
19854 }
19855
19856 #[cfg(unix)]
19859 #[tokio::test]
19860 async fn verification_is_standalone_inside_a_live_worker() {
19861 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19862 if std::env::var_os(CHILD).is_none() {
19863 let home = tempfile::tempdir().unwrap();
19864 let generation = crate::worker::pin_current(home.path()).unwrap();
19865 let output = std::process::Command::new(std::env::current_exe().unwrap())
19866 .args([
19867 "--exact",
19868 "graph::tests::verification_is_standalone_inside_a_live_worker",
19869 "--nocapture",
19870 ])
19871 .env(CHILD, "1")
19872 .env("MAGI_HOME", home.path())
19873 .env("MAGI_WORKER_HOME", home.path())
19874 .env("MAGI_WORKER_ID", "abcdef")
19875 .env("MAGI_WORKER_GENERATION", &generation.id)
19876 .env("MAGI_WORKER_EXE", &generation.executable)
19877 .env("MAGI_RUN", "verification-parent")
19878 .env("MAGI_NODE", "gate")
19879 .env("MAGI_SEAT", "worker-fixture")
19880 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19881 .env("MAGI_NO_AUTOUPDATE", "0")
19882 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19883 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19884 .env(
19885 "VERIFY_EXPECT_TOOLCHAIN",
19886 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19887 )
19888 .env(
19889 "VERIFY_EXPECT_CACHE",
19890 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19891 )
19892 .output()
19893 .unwrap();
19894 assert!(
19895 output.status.success()
19896 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19897 "{}\n{}",
19898 String::from_utf8_lossy(&output.stdout),
19899 String::from_utf8_lossy(&output.stderr)
19900 );
19901 return;
19902 }
19903 let home = crate::run::home();
19904 let directory = home.join("workers/jobs/abcdef");
19905 std::fs::create_dir_all(&directory).unwrap();
19906 let job = crate::worker::Job {
19907 id: "abcdef".into(),
19908 sequence: 1,
19909 kind: "task".into(),
19910 key: "task:verification".into(),
19911 payload: serde_json::json!({}),
19912 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19913 version: Some(env!("CARGO_PKG_VERSION").into()),
19914 status: crate::worker::Status::Running,
19915 result: None,
19916 pid: Some(std::process::id()),
19917 created_at: 0,
19918 finished_at: None,
19919 };
19920 std::fs::write(
19921 directory.join("job.json"),
19922 serde_json::to_vec(&job).unwrap(),
19923 )
19924 .unwrap();
19925 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19926 let assert_worker = || {
19927 assert_eq!(
19928 crate::worker::current_job(&home).unwrap().unwrap().id,
19929 job.id
19930 );
19931 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19932 assert_eq!(
19933 crate::worker::helper_executable(&home).unwrap(),
19934 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19935 );
19936 assert!(
19937 crate::updater::Checker::new(&crate::config::Update {
19938 mode: crate::config::UpdateMode::Notify,
19939 interval: None,
19940 auto_restart: false,
19941 })
19942 .is_none(),
19943 "the driver remains pinned, even while tests are standalone"
19944 );
19945 };
19946 assert_worker();
19947 let repo = home.join("repo");
19948 std::fs::create_dir_all(&repo).unwrap();
19949 init_repo(&repo);
19950 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19953 test -z "${MAGI_WORKER_GENERATION+x}" &&
19954 test -z "${MAGI_WORKER_HOME+x}" &&
19955 test -z "${MAGI_WORKER_EXE+x}" &&
19956 test "$MAGI_HOME/repo" -ef . &&
19957 test "$MAGI_RUN" = verification-parent &&
19958 test "$MAGI_NODE" = gate &&
19959 test "$MAGI_SEAT" = worker-fixture &&
19960 test -n "$MAGI_PROCESS_OWNER" &&
19961 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19962 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19963 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19964 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19965 assert!(
19967 !std::process::Command::new("sh")
19968 .args(["-c", probe])
19969 .current_dir(&repo)
19970 .status()
19971 .unwrap()
19972 .success()
19973 );
19974 let unrelated_home = tempfile::tempdir().unwrap();
19975 assert!(
19976 crate::worker::current_job(unrelated_home.path()).is_err(),
19977 "an inherited worker id cannot be resolved in a unit fixture's home"
19978 );
19979 let mut runner = runner_at(RunStatus::Gating);
19980 runner.state.repo = repo.clone();
19981 runner.state.config.disk.min_free_bytes = 0;
19982 runner.state.config.coordination = crate::config::test_coordination();
19983 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19984 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19985 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19986 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19987 let winner = Candidate {
19988 index: 0,
19989 label: 'A',
19990 agent: "fixture".into(),
19991 branch: "main".into(),
19992 worktree: repo.clone(),
19993 summary: String::new(),
19994 stat: String::new(),
19995 files: 0,
19996 commits: 0,
19997 empty: false,
19998 failed: None,
19999 verified_noop: None,
20000 duration_ms: 0,
20001 folded: false,
20002 };
20003 let (e2e, retried, pids) = run_e2e_with_retry(
20004 &mut runner.state,
20005 &["sh".into(), "-c".into()],
20006 &[format!("{probe} && printf initial")],
20007 &repo,
20008 Duration::from_secs(10),
20009 "initial",
20010 )
20011 .await;
20012 assert!(!retried && pids.is_empty());
20013 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20014 assert_eq!(e2e[0].output_tail, "initial");
20015 let gate = runner.run_gate(&winner).await.unwrap();
20016 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20017 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20020 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20021 passed: false,
20022 outcomes: Vec::new(),
20023 });
20024 runner.state.save().unwrap();
20025 let mut resumed = RunState::load(&runner.state.id).unwrap();
20026 let wait = crate::repo_lock::WaitPolicy::default();
20027 assert!(matches!(
20028 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20029 .await
20030 .unwrap(),
20031 GateFix::Retry
20032 ));
20033 let saved = RunState::load(&resumed.id).unwrap();
20034 let repair = saved.gate_repair_e2e.unwrap();
20035 assert!(repair.passed);
20036 assert_eq!(repair.outcomes.len(), 1);
20037 assert!(repair.outcomes[0].ok());
20038 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20039 .await
20040 .unwrap();
20041 assert_eq!(landing.len(), 1);
20042 assert!(landing[0].ok(), "{landing:?}");
20043 assert_eq!(
20044 std::fs::read_to_string(repo.join("phases")).unwrap(),
20045 "pre\ngate\ne2e\npre\ngate\n"
20046 );
20047 assert_worker();
20048 }
20049
20050 #[cfg(unix)]
20051 #[tokio::test]
20052 async fn verification_keeps_the_panic_and_the_full_output() {
20053 crate::run::pin_test_home();
20054 let tmp = tempfile::tempdir().unwrap();
20055 let mut config = Config::default();
20056 config.disk.min_free_bytes = 0;
20057 let mut state = RunState::new(
20058 tmp.path().to_owned(),
20059 "main".into(),
20060 "base".into(),
20061 "verification evidence".into(),
20062 config,
20063 );
20064 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20065 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20066 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20067 let mut names = Vec::new();
20068 for _ in 0..2 {
20069 let (outcomes, _) = run_commands(
20070 &mut state,
20071 "gate",
20072 "gate",
20073 0,
20074 &["sh".into(), "-c".into()],
20075 &[script.into()],
20076 tmp.path(),
20077 Duration::from_secs(60),
20078 )
20079 .await;
20080 let tail = &outcomes[0].output_tail;
20081 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20082 assert!(tail.contains("boom assertion"));
20083 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20084 let line = tail.lines().next().unwrap();
20085 names.push(line.to_owned());
20086 }
20087 assert_ne!(names[0], names[1], "the second run must not overwrite");
20088 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20089 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20090 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20091
20092 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20094 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20095 let (outcomes, _) = run_commands(
20096 &mut state,
20097 "gate",
20098 "gate",
20099 0,
20100 &["sh".into(), "-c".into()],
20101 &["echo failure-output; exit 7".into()],
20102 tmp.path(),
20103 Duration::from_secs(10),
20104 )
20105 .await;
20106 assert_eq!(outcomes[0].code, Some(7));
20107 assert!(state.events.iter().any(|e| {
20108 e.node == "gate" && e.message.contains("could not save the full gate output:")
20109 }));
20110 }
20111
20112 #[cfg(unix)]
20113 #[tokio::test]
20114 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20115 crate::run::pin_test_home();
20116 let tmp = tempfile::tempdir().unwrap();
20117 let mut config = Config::default();
20118 config.disk.min_free_bytes = 0;
20119 let mut state = RunState::new(
20120 tmp.path().to_owned(),
20121 "main".into(),
20122 "base".into(),
20123 "verification evidence".into(),
20124 config,
20125 );
20126 let (outcomes, pids) = run_commands(
20127 &mut state,
20128 "gate",
20129 "gate",
20130 1,
20131 &["sh".into(), "-c".into()],
20132 &[
20133 "printf before-timeout; exec sleep 30".into(),
20134 "kill -TERM $$".into(),
20135 "exit 7".into(),
20136 "printf success".into(),
20137 ],
20138 tmp.path(),
20139 Duration::from_secs(1),
20140 )
20141 .await;
20142 wait_for_timed_out_children_to_die(&pids).await;
20143 assert_eq!(
20144 pids.len(),
20145 1,
20146 "unleased timeout retains its death-check PID"
20147 );
20148 assert_eq!(outcomes.len(), 4);
20149 assert_eq!(outcomes[0].code, None);
20150 assert!(outcomes[0].output_tail.contains("before-timeout"));
20151 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20152 assert_eq!(outcomes[1].code, None);
20153 assert!(outcomes[1].output_tail.contains("command terminated:"));
20154 assert!(!outcomes[1].output_tail.contains("timed out"));
20155 assert_eq!(outcomes[2].code, Some(7));
20156 assert!(!outcomes[2].output_tail.contains("timed out"));
20157 assert_eq!(outcomes[3].code, Some(0));
20158 assert_eq!(outcomes[3].output_tail, "success");
20159 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20160 }
20161
20162 #[tokio::test]
20172 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20173 crate::run::pin_test_home();
20174
20175 let tmp = tempfile::tempdir().expect("tempdir");
20176 let repo = tmp.path().join("repo");
20177 std::fs::create_dir_all(&repo).unwrap();
20178 init_repo(&repo);
20179
20180 let mut config = Config::default();
20181 config.disk.min_free_bytes = 0;
20182 config.verify.gate = vec![
20183 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20184 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20185 .to_owned(),
20186 ];
20187
20188 let mut state = RunState::new(
20189 repo.clone(),
20190 "main".to_owned(),
20191 "deadbeef".to_owned(),
20192 "task".to_owned(),
20193 config,
20194 );
20195 let run_id = state.id.clone();
20196 state.candidates = vec![Candidate {
20197 index: 0,
20198 label: 'A',
20199 agent: "alpha".to_owned(),
20200 branch: "does-not-exist".to_owned(),
20201 worktree: repo.clone(),
20202 summary: String::new(),
20203 stat: String::new(),
20204 files: 0,
20205 commits: 0,
20206 empty: false,
20207 failed: None,
20208 verified_noop: None,
20209 duration_ms: 0,
20210 folded: false,
20211 }];
20212 state.tally = Some(Tally {
20213 first_choice: BTreeMap::from([('A', 1)]),
20214 borda: BTreeMap::new(),
20215 winner: 'A',
20216 rankings: 1,
20217 unanimous_initial: true,
20218 deliberated: false,
20219 changed_votes: 0,
20220 unanimous_final: true,
20221 tie_break: None,
20222 judges: 0,
20223 present: 0,
20224 quorum: 0,
20225 met_quorum: true,
20226 uncontested: Some("only candidate A produced a change".to_owned()),
20227 });
20228 state.reviews = vec![ReviewRound {
20229 round: 1,
20230 head: "deadbeef".to_owned(),
20231 verified_head: None,
20232 verified_at: None,
20233 reviews: Vec::new(),
20234 e2e: Vec::new(),
20235 fix: None,
20236 blocking: 0,
20237 answered: 0,
20238 expected: 0,
20239 clean: true,
20240 verify_retried: false,
20241 e2e_deferred: false,
20242 e2e_defer_reason: None,
20243 progressed: false,
20244 vote_split: false,
20245 reconsideration: Vec::new(),
20246 verdict: None,
20247 }];
20248
20249 let mut runner = Runner {
20250 state,
20251 roles: ResolvedRoles {
20252 implementers: Vec::new(),
20253 judges: Vec::new(),
20254 reviewers: Vec::new(),
20255 fixer: None,
20256 conductor: conductor(),
20257 implementer_roster: Vec::new(),
20258 judge_roster: Vec::new(),
20259 reviewer_roster: Vec::new(),
20260 },
20261 sem: Arc::new(Semaphore::new(1)),
20262 pause: Pause::new(),
20263 interrupt: Pause::new(),
20264 };
20265
20266 let started_marker = repo.join("started.marker");
20267 let release_marker = repo.join("release.marker");
20268 let poller = tokio::spawn(async move {
20269 for _ in 0..100 {
20274 if started_marker.exists()
20275 && let Ok(s) = crate::run::RunState::load(&run_id)
20276 && let Some(a) = s.active.get("gate")
20277 {
20278 std::fs::write(&release_marker, b"go").expect("release marker");
20279 return Some(a.clone());
20280 }
20281 tokio::time::sleep(Duration::from_millis(50)).await;
20282 }
20283 None
20284 });
20285
20286 runner.gate().await.expect("gate");
20287 let captured = poller.await.expect("poller task");
20288 let captured = captured.expect(
20289 "the poller never saw a `gate` task entry in run.json while the command was \
20290 still blocked on its own release marker",
20291 );
20292
20293 assert_eq!(captured.task.as_deref(), Some("gate"));
20294 assert_eq!(captured.node, "gate");
20295 assert_eq!(captured.index, Some(1));
20296 assert_eq!(captured.total, Some(1));
20297 assert!(
20298 captured
20299 .command
20300 .as_deref()
20301 .is_some_and(|c| c.contains("started.marker")),
20302 "{captured:?}"
20303 );
20304
20305 assert!(
20306 runner.state.active.is_empty(),
20307 "the entry must be cleared once the command actually finished: {:?}",
20308 runner.state.active
20309 );
20310 assert!(runner.state.gate_ran);
20311 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20312 }
20313
20314 #[tokio::test]
20317 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20318 use crate::verdict::{Finding, ReviewVote, Severity};
20319 crate::run::pin_test_home();
20320 let tmp = tempfile::tempdir().expect("tempdir");
20321 let repo = tmp.path().join("repo");
20322 std::fs::create_dir_all(&repo).unwrap();
20323 init_repo(&repo);
20324
20325 for (severity, vote, expect) in [
20326 (Severity::Major, ReviewVote::Reject, true),
20327 (Severity::Minor, ReviewVote::Reject, false),
20328 (Severity::Major, ReviewVote::Approve, false),
20329 ] {
20330 let mut round = review_round(false, 1, 1, 1, false, true);
20331 round.reviews = vec![ReviewRecord {
20332 reviewer: 1,
20333 agent: "alpha".to_owned(),
20334 summary: String::new(),
20335 findings: vec![Finding {
20336 id: "R1-1-1".to_owned(),
20337 severity,
20338 file: None,
20339 line: None,
20340 title: "t".to_owned(),
20341 detail: String::new(),
20342 }],
20343 vote: Some(vote),
20344 failed: None,
20345 duration_ms: 0,
20346 attempts: 0,
20347 }];
20348 let mut state = RunState::new(
20349 repo.clone(),
20350 "main".to_owned(),
20351 "deadbeef".to_owned(),
20352 "task".to_owned(),
20353 Config::default(),
20354 );
20355 state.reviews = vec![round];
20356 let mut runner = Runner {
20357 state,
20358 roles: ResolvedRoles {
20359 implementers: Vec::new(),
20360 judges: Vec::new(),
20361 reviewers: Vec::new(),
20362 fixer: None,
20363 conductor: conductor(),
20364 implementer_roster: Vec::new(),
20365 judge_roster: Vec::new(),
20366 reviewer_roster: Vec::new(),
20367 },
20368 sem: Arc::new(Semaphore::new(1)),
20369 pause: Pause::new(),
20370 interrupt: Pause::new(),
20371 };
20372 let shell = runner.state.config.shell();
20373 runner
20374 .stop_reviewing("round budget spent", &shell, &repo)
20375 .await
20376 .expect("stop_reviewing");
20377 assert_eq!(runner.state.status, RunStatus::Gating);
20378 assert_eq!(
20379 runner.state.contested_handoff.is_some(),
20380 expect,
20381 "{severity:?} + {vote:?}"
20382 );
20383 }
20384 }
20385
20386 #[tokio::test]
20399 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20400 crate::run::pin_test_home();
20401 let home = crate::run::home();
20402
20403 let tmp = tempfile::tempdir().expect("tempdir");
20404 let repo = tmp.path().join("repo");
20405 std::fs::create_dir_all(&repo).unwrap();
20406 init_repo(&repo);
20407 let head = crate::git::rev_parse(&repo, "HEAD")
20408 .await
20409 .expect("rev-parse");
20410 let cache_dir = tmp.path().join("target");
20413
20414 let mut config = Config::default();
20415 config.verify.e2e = vec![format!(
20416 "CARGO_TARGET_DIR='{}' test -f README.md",
20417 cache_dir.display()
20418 )];
20419 config.graph.review_rounds = 1;
20420 config.graph.timeout_verify = Some(2);
20423 config.graph.timeout_resource_wait = 2;
20424
20425 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20426 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20427 .expect("no io error acquiring directly")
20428 {
20429 crate::cache::AcquireOutcome::Acquired(g) => g,
20430 crate::cache::AcquireOutcome::Busy(b) => {
20431 panic!("expected the direct acquire to win the lease first: {b:?}")
20432 }
20433 };
20434
20435 let mut state = RunState::new(
20436 repo.clone(),
20437 "main".to_owned(),
20438 head.clone(),
20439 "task".to_owned(),
20440 config,
20441 );
20442 state.candidates = vec![Candidate {
20443 index: 0,
20444 label: 'A',
20445 agent: "alpha".to_owned(),
20446 branch: "does-not-exist".to_owned(),
20447 worktree: repo.clone(),
20448 summary: String::new(),
20449 stat: String::new(),
20450 files: 0,
20451 commits: 0,
20452 empty: false,
20453 failed: None,
20454 verified_noop: None,
20455 duration_ms: 0,
20456 folded: false,
20457 }];
20458 state.tally = Some(Tally {
20459 first_choice: BTreeMap::from([('A', 1)]),
20460 borda: BTreeMap::new(),
20461 winner: 'A',
20462 rankings: 1,
20463 unanimous_initial: true,
20464 deliberated: false,
20465 changed_votes: 0,
20466 unanimous_final: true,
20467 tie_break: None,
20468 judges: 0,
20469 present: 0,
20470 quorum: 0,
20471 met_quorum: true,
20472 uncontested: Some("only candidate A produced a change".to_owned()),
20473 });
20474 state.reviews = vec![ReviewRound {
20478 round: 1,
20479 head: head.clone(),
20480 verified_head: None,
20481 verified_at: None,
20482 reviews: Vec::new(),
20483 e2e: Vec::new(),
20484 fix: None,
20485 blocking: 1,
20486 answered: 1,
20487 expected: 1,
20488 clean: false,
20489 verify_retried: false,
20490 e2e_deferred: true,
20491 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20492 progressed: false,
20493 vote_split: false,
20494 reconsideration: Vec::new(),
20495 verdict: None,
20496 }];
20497
20498 let mut runner = Runner {
20499 state,
20500 roles: ResolvedRoles {
20501 implementers: Vec::new(),
20502 judges: Vec::new(),
20503 reviewers: Vec::new(),
20504 fixer: None,
20505 conductor: conductor(),
20506 implementer_roster: Vec::new(),
20507 judge_roster: Vec::new(),
20508 reviewer_roster: Vec::new(),
20509 },
20510 sem: Arc::new(Semaphore::new(1)),
20511 pause: Pause::new(),
20512 interrupt: Pause::new(),
20513 };
20514
20515 let shell = runner.state.config.shell();
20516 runner
20517 .stop_reviewing("round budget spent", &shell, &repo)
20518 .await
20519 .expect("stop_reviewing");
20520
20521 let last = runner.state.reviews.last().expect("round record");
20522 assert_eq!(
20523 last.e2e_status(),
20524 E2eStatus::ResourceBlocked,
20525 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20526 failed: {last:?}"
20527 );
20528 assert_eq!(
20529 last.verified_head.as_deref(),
20530 Some(head.as_str()),
20531 "which commit this attempt targeted is known even though nothing finished checking \
20532 it"
20533 );
20534 let first_attempt_at = last
20535 .verified_at
20536 .expect("when this attempt ran is known too");
20537 assert_ne!(
20538 runner.state.status,
20539 RunStatus::Blocked,
20540 "contention is evidence about the machine, not the patch — it must not settle the \
20541 run as blocked: {:?}",
20542 runner.state.status
20543 );
20544 assert!(
20545 !runner
20546 .state
20547 .events
20548 .iter()
20549 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20550 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20551 runner.state.events
20552 );
20553
20554 runner
20559 .stop_reviewing("round budget spent", &shell, &repo)
20560 .await
20561 .expect("stop_reviewing retry");
20562 assert_eq!(
20563 runner.state.reviews.len(),
20564 1,
20565 "no new round was started: {:?}",
20566 runner.state.reviews
20567 );
20568 let last = runner.state.reviews.last().expect("round record");
20569 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20570 assert!(
20571 last.verified_at.expect("still known") > first_attempt_at,
20572 "a second reentry must be a fresh attempt, not a stale copy of the first"
20573 );
20574 assert_ne!(runner.state.status, RunStatus::Blocked);
20575
20576 held.release();
20577 }
20578
20579 #[cfg(unix)]
20580 #[tokio::test]
20581 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20582 ask_test_home();
20583 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20584 let tmp = tempfile::tempdir().unwrap();
20585 let repo = tmp.path().join("repo");
20586 std::fs::create_dir(&repo).unwrap();
20587 init_repo(&repo);
20588 let run_git = |args: &[&str]| {
20589 let out = std::process::Command::new("git")
20590 .args(args)
20591 .current_dir(&repo)
20592 .output()
20593 .unwrap();
20594 assert!(
20595 out.status.success(),
20596 "{args:?}: {}",
20597 String::from_utf8_lossy(&out.stderr)
20598 );
20599 };
20600 let origin = tmp.path().join("origin.git");
20601 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20602 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20603 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20604 run_git(&["add", "spec.rs"]);
20605 run_git(&["commit", "-m", "initial spec"]);
20606 run_git(&["push", "origin", "main"]);
20607 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20608 run_git(&["checkout", "-b", "fix/review"]);
20609 std::fs::write(
20610 repo.join("check.rs"),
20611 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20612 )
20613 .unwrap();
20614 run_git(&["add", "check.rs"]);
20615 run_git(&["commit", "-m", "use spec"]);
20616 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20617 run_git(&["checkout", "main"]);
20618 std::fs::write(
20619 repo.join("spec.rs"),
20620 "struct AgentSpec { max_turn_secs: u64 }\n",
20621 )
20622 .unwrap();
20623 run_git(&["add", "spec.rs"]);
20624 run_git(&["commit", "-m", "extend spec"]);
20625 run_git(&["push", "origin", "main"]);
20626 run_git(&["checkout", "fix/review"]);
20627
20628 let mut runner = runner_at(RunStatus::Reviewing);
20629 runner.state.repo = repo.clone();
20630 runner.state.base_commit = base;
20631 runner.state.pr_head = Some("fix/review".into());
20632 runner.state.config.graph.review_rounds = max_rounds;
20633 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20634 runner.state.config.graph.sessions = false;
20635 runner.state.config.disk.min_free_bytes = 0;
20636 runner.state.config.verify.e2e =
20637 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20638 runner.state.candidates = vec![Candidate {
20639 index: 0,
20640 label: 'A',
20641 agent: "fixture".into(),
20642 branch: "fix/review".into(),
20643 worktree: repo.clone(),
20644 summary: String::new(),
20645 stat: String::new(),
20646 files: 1,
20647 commits: 1,
20648 empty: false,
20649 failed: None,
20650 verified_noop: None,
20651 duration_ms: 0,
20652 folded: false,
20653 }];
20654 runner.state.tally = Some(Tally {
20655 first_choice: BTreeMap::new(),
20656 borda: BTreeMap::new(),
20657 winner: 'A',
20658 rankings: 0,
20659 unanimous_initial: true,
20660 deliberated: false,
20661 changed_votes: 0,
20662 unanimous_final: true,
20663 tie_break: None,
20664 judges: 0,
20665 present: 0,
20666 quorum: 0,
20667 met_quorum: true,
20668 uncontested: Some("review-only".into()),
20669 });
20670 let mut last = review_round(false, 0, 1, 1, false, false);
20671 last.head = reviewed.clone();
20672 last.e2e_deferred = true;
20673 last.e2e[0].resource_blocked = true;
20674 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20675 if max_rounds == 1 {
20676 last.e2e.clear();
20677 }
20678 runner.state.reviews = vec![last];
20679 let calls = tmp.path().join("fix-calls");
20680 let mut fixer = spec("repair");
20681 fixer
20682 .env
20683 .insert("FIX_CALLS".into(), calls.display().to_string());
20684 fixer.command = vec!["sh".into(), "-c".into(), r#"
20685 echo fix >> "$FIX_CALLS"
20686 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20687 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20688 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20689 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20690 "#.into()];
20691 let mut reviewer = spec("reviewer");
20692 reviewer.command = vec![
20693 "sh".into(),
20694 "-c".into(),
20695 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20696 .into(),
20697 ];
20698 runner.roles.fixer = Some(vec![fixer]);
20699 runner.roles.reviewers = vec![reviewer];
20700
20701 runner.sync_to_base().await.unwrap();
20702 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20703 assert_ne!(rebased, reviewed);
20704 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20705 if reentry {
20706 let shell = runner.state.config.shell();
20709 runner
20710 .stop_unresolved_review("deferred verification", &shell, &repo)
20711 .await
20712 .unwrap();
20713 assert!(runner.catchup_repairable());
20714 assert!(!calls.exists());
20715 runner.state = RunState::load(&runner.state.id).unwrap();
20716 assert!(runner.catchup_repairable());
20717 }
20718 runner.review_loop().await.unwrap();
20719 let first = &runner.state.reviews[0];
20720 assert_eq!(first.head, reviewed);
20721 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20722 assert!(
20723 first.verify_retried,
20724 "a reproduced build failure gets exactly one retry: {:?}",
20725 first.e2e
20726 );
20727 assert!(first.e2e[0].output_tail.contains("E0063"));
20728 assert!(!runner.catchup_repairable());
20729 if max_rounds == 2 {
20730 assert_eq!(
20731 runner.state.status,
20732 RunStatus::Gating,
20733 "{}",
20734 crate::report::run(&runner.state)
20735 );
20736 assert_eq!(runner.state.reviews.len(), 2);
20737 assert!(first.fix.as_ref().unwrap().committed);
20738 assert!(runner.state.reviews[1].clean);
20739 assert_eq!(
20740 runner
20741 .state
20742 .reviews
20743 .iter()
20744 .filter(|r| r.fix.is_some())
20745 .count(),
20746 1
20747 );
20748 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20749 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20750 } else {
20751 assert_eq!(runner.state.status, RunStatus::Blocked);
20752 assert!(first.fix.is_none());
20753 assert!(!calls.exists());
20754 let recovery = crate::recovery::assess(
20755 "prior",
20756 Some(&runner.state),
20757 &runner.landing_base(),
20758 crate::recovery::Relation::Diverged,
20759 );
20760 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20761 let reasons = recovery.reasons.join("\n");
20762 assert!(
20763 reasons.contains(&reviewed) && reasons.contains(&rebased),
20764 "{reasons}"
20765 );
20766 assert!(
20767 reasons.contains("base sync to")
20768 && reasons.contains("additional review is required"),
20769 "{reasons}"
20770 );
20771 }
20772 }
20773 }
20774
20775 #[tokio::test]
20787 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20788 crate::run::pin_test_home();
20789 let home = crate::run::home();
20790
20791 let tmp = tempfile::tempdir().expect("tempdir");
20792 let repo = tmp.path().join("repo");
20793 std::fs::create_dir_all(&repo).unwrap();
20794 init_repo(&repo);
20795 let head = crate::git::rev_parse(&repo, "HEAD")
20796 .await
20797 .expect("rev-parse");
20798 let cache_dir = tmp.path().join("target");
20799
20800 let mut config = Config::default();
20801 config.verify.e2e = vec![format!(
20802 "CARGO_TARGET_DIR='{}' test -f README.md",
20803 cache_dir.display()
20804 )];
20805 config.graph.review_rounds = 1;
20806 config.graph.timeout_verify = Some(2);
20807 config.graph.timeout_resource_wait = 2;
20808
20809 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20810 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20811 .expect("no io error acquiring directly")
20812 {
20813 crate::cache::AcquireOutcome::Acquired(g) => g,
20814 crate::cache::AcquireOutcome::Busy(b) => {
20815 panic!("expected the direct acquire to win the lease first: {b:?}")
20816 }
20817 };
20818
20819 let mut state = RunState::new(
20820 repo.clone(),
20821 "main".to_owned(),
20822 head.clone(),
20823 "task".to_owned(),
20824 config,
20825 );
20826 state.candidates = vec![Candidate {
20827 index: 0,
20828 label: 'A',
20829 agent: "alpha".to_owned(),
20830 branch: "does-not-exist".to_owned(),
20831 worktree: repo.clone(),
20832 summary: String::new(),
20833 stat: String::new(),
20834 files: 0,
20835 commits: 0,
20836 empty: false,
20837 failed: None,
20838 verified_noop: None,
20839 duration_ms: 0,
20840 folded: false,
20841 }];
20842 state.tally = Some(Tally {
20843 first_choice: BTreeMap::from([('A', 1)]),
20844 borda: BTreeMap::new(),
20845 winner: 'A',
20846 rankings: 1,
20847 unanimous_initial: true,
20848 deliberated: false,
20849 changed_votes: 0,
20850 unanimous_final: true,
20851 tie_break: None,
20852 judges: 0,
20853 present: 0,
20854 quorum: 0,
20855 met_quorum: true,
20856 uncontested: Some("only candidate A produced a change".to_owned()),
20857 });
20858 state.reviews = vec![ReviewRound {
20862 round: 1,
20863 head: head.clone(),
20864 verified_head: Some(head.clone()),
20865 verified_at: Some(jiff::Timestamp::now()),
20866 reviews: Vec::new(),
20867 e2e: vec![CommandOutcome {
20868 command: format!(
20869 "CARGO_TARGET_DIR='{}' test -f README.md",
20870 cache_dir.display()
20871 ),
20872 code: None,
20873 output_tail: "waiting for the shared build cache".to_owned(),
20874 duration_ms: 0,
20875 resource_blocked: true,
20876 }],
20877 fix: None,
20878 blocking: 1,
20879 answered: 1,
20880 expected: 1,
20881 clean: false,
20882 verify_retried: false,
20883 e2e_deferred: false,
20884 e2e_defer_reason: None,
20885 progressed: false,
20886 vote_split: false,
20887 reconsideration: Vec::new(),
20888 verdict: None,
20889 }];
20890
20891 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20892 let mut runner = Runner {
20893 state,
20894 roles: ResolvedRoles {
20895 implementers: Vec::new(),
20896 judges: Vec::new(),
20897 reviewers: Vec::new(),
20898 fixer: None,
20899 conductor: conductor(),
20900 implementer_roster: Vec::new(),
20901 judge_roster: Vec::new(),
20902 reviewer_roster: Vec::new(),
20903 },
20904 sem: Arc::new(Semaphore::new(1)),
20905 pause: Pause::new(),
20906 interrupt: Pause::new(),
20907 };
20908
20909 runner.review_loop().await.expect("review_loop");
20914
20915 assert_eq!(
20916 runner.state.reviews.len(),
20917 1,
20918 "no new round was started on top of the unresolved one: {:?}",
20919 runner.state.reviews
20920 );
20921 let last = &runner.state.reviews[0];
20922 assert_eq!(
20923 last.e2e_status(),
20924 E2eStatus::ResourceBlocked,
20925 "still contended: {last:?}"
20926 );
20927 assert!(
20928 last.verified_at.expect("still known") > first_attempt_at,
20929 "review_loop must have actually retried the check, not left it exactly as found"
20930 );
20931 assert_ne!(
20932 runner.state.status,
20933 RunStatus::Blocked,
20934 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20935 runner.state.status
20936 );
20937
20938 held.release();
20939 }
20940
20941 #[tokio::test]
20942 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20943 crate::run::pin_test_home();
20944 for status in [RunStatus::Blocked, RunStatus::Gating] {
20945 let mut runner = runner_at(status);
20946 runner.state.config.graph.land = true;
20947 runner.state.gate_ran = true;
20948 runner.state.merge = Some(MergeOutcome {
20949 mode: MergeMode::Pr,
20950 ok: false,
20951 detail: "the fixer produced no commit".into(),
20952 empty: false,
20953 });
20954 let url = "https://example.invalid/o/r/pull/1";
20955 runner.state.pr = Some(crate::run::PrRecord {
20956 url: url.into(),
20957 number: 1,
20958 state: "open".into(),
20959 checks: "red".into(),
20960 round: 1,
20961 rounds: 1,
20962 red_at_merge: Vec::new(),
20963 });
20964 runner.state.save().unwrap();
20965 runner.execute_graph().await.unwrap();
20969 assert_eq!(runner.state.status, RunStatus::Blocked);
20970 assert!(runner.state.land_sync_pending);
20971 assert!(runner.state.events.iter().any(|event|
20972 event.node == "land" && event.message == format!("watching {url}")));
20973 assert_eq!(
20974 runner.state.merge.as_ref().unwrap().detail,
20975 "the fixer produced no commit"
20976 );
20977 }
20978 }
20979
20980 #[tokio::test]
20981 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20982 crate::run::pin_test_home();
20983 let tmp = tempfile::tempdir().expect("tempdir");
20984 let repo = tmp.path().join("repo");
20985 std::fs::create_dir_all(&repo).unwrap();
20986 init_repo(&repo);
20987
20988 let mut config = Config::default();
20989 config.merge.mode = MergeMode::Pr;
20990 config.graph.land = true;
20991 config.graph.land_approval = false;
20992
20993 let mut state = RunState::new(
20994 repo.clone(),
20995 "main".to_owned(),
20996 "deadbeef".to_owned(),
20997 "task".to_owned(),
20998 config,
20999 );
21000 state.candidates = vec![Candidate {
21001 index: 0,
21002 label: 'A',
21003 agent: "alpha".to_owned(),
21004 branch: "does-not-exist".to_owned(),
21005 worktree: repo.clone(),
21006 summary: String::new(),
21007 stat: String::new(),
21008 files: 0,
21009 commits: 0,
21010 empty: false,
21011 failed: None,
21012 verified_noop: None,
21013 duration_ms: 0,
21014 folded: false,
21015 }];
21016 state.tally = Some(Tally {
21017 first_choice: BTreeMap::from([('A', 1)]),
21018 borda: BTreeMap::new(),
21019 winner: 'A',
21020 rankings: 1,
21021 unanimous_initial: true,
21022 deliberated: false,
21023 changed_votes: 0,
21024 unanimous_final: true,
21025 tie_break: None,
21026 judges: 0,
21027 present: 0,
21028 quorum: 0,
21029 met_quorum: true,
21030 uncontested: Some("only candidate A produced a change".to_owned()),
21031 });
21032 state.reviews = vec![ReviewRound {
21033 round: 1,
21034 head: "deadbeef".to_owned(),
21035 verified_head: None,
21036 verified_at: None,
21037 reviews: Vec::new(),
21038 e2e: Vec::new(),
21039 fix: None,
21040 blocking: 0,
21041 answered: 0,
21042 expected: 0,
21043 clean: true,
21044 verify_retried: false,
21045 e2e_deferred: false,
21046 e2e_defer_reason: None,
21047 progressed: false,
21048 vote_split: false,
21049 reconsideration: Vec::new(),
21050 verdict: None,
21051 }];
21052 state.gate = vec![CommandOutcome {
21053 command: "test".to_owned(),
21054 code: Some(0),
21055 output_tail: String::new(),
21056 duration_ms: 0,
21057 resource_blocked: false,
21058 }];
21059 state.gate_ran = true;
21060 state.status = RunStatus::Landing;
21064 state.merge = Some(MergeOutcome {
21065 mode: MergeMode::Pr,
21066 ok: true,
21067 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21068 empty: false,
21069 });
21070
21071 ask_test_home();
21075 state.save().expect("save landing run");
21078 let store = ask::Questions::open();
21079 let q = ask_open_question(&store, &state.id);
21080
21081 let mut runner = Runner {
21082 state,
21083 roles: ResolvedRoles {
21084 implementers: Vec::new(),
21085 judges: Vec::new(),
21086 reviewers: Vec::new(),
21087 fixer: None,
21088 conductor: conductor(),
21089 implementer_roster: Vec::new(),
21090 judge_roster: Vec::new(),
21091 reviewer_roster: Vec::new(),
21092 },
21093 sem: Arc::new(Semaphore::new(1)),
21094 pause: Pause::new(),
21095 interrupt: Pause::new(),
21096 };
21097
21098 runner.execute().await.expect("execute");
21103
21104 assert_eq!(
21105 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21106 Some("https://example.invalid/x/y/pull/1"),
21107 "reentry must not push again or open a second pull request over the \
21108 one `land` is already watching"
21109 );
21110 assert_ne!(
21111 runner.state.status,
21112 RunStatus::Landing,
21113 "land could not actually reach the fake pull request, so it must \
21114 have given up rather than left the run silently parked forever"
21115 );
21116 assert_eq!(runner.state.status, RunStatus::Blocked);
21120 assert!(
21121 store.get(&q.id).unwrap().status.open(),
21122 "Blocked is still alive; settle_questions must have been a no-op here"
21123 );
21124 }
21125
21126 fn state_with_round(round: ReviewRound) -> RunState {
21127 let mut s = RunState::new(
21128 PathBuf::from("/repo"),
21129 "main".to_owned(),
21130 "abc1234".to_owned(),
21131 "add retries".to_owned(),
21132 Config::default(),
21133 );
21134 s.reviews = vec![round];
21135 s
21136 }
21137
21138 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21139 crate::verdict::Finding {
21140 id: id.to_owned(),
21141 severity,
21142 file: None,
21143 line: None,
21144 title: title.to_owned(),
21145 detail: String::new(),
21146 }
21147 }
21148
21149 #[test]
21150 fn pr_body_names_open_findings_and_declined_ones() {
21151 let round = ReviewRound {
21152 round: 2,
21153 head: "deadbee".to_owned(),
21154 verified_head: None,
21155 verified_at: None,
21156 reviews: vec![ReviewRecord {
21157 attempts: 0,
21158 reviewer: 1,
21159 agent: "alpha".to_owned(),
21160 summary: String::new(),
21161 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21162 vote: None,
21163 failed: None,
21164 duration_ms: 0,
21165 }],
21166 e2e: vec![CommandOutcome {
21167 command: "cargo test".to_owned(),
21168 code: Some(0),
21169 output_tail: String::new(),
21170 duration_ms: 0,
21171 resource_blocked: false,
21172 }],
21173 verify_retried: false,
21174 e2e_deferred: false,
21175 e2e_defer_reason: None,
21176 fix: Some(FixRecord {
21177 agent: "alpha".to_owned(),
21178 addressed: Vec::new(),
21179 rejected: vec![crate::verdict::Rejection {
21180 id: "R1-1-1".to_owned(),
21181 why: "not reachable from any caller".to_owned(),
21182 }],
21183 notes: String::new(),
21184 committed: true,
21185 failed: None,
21186 duration_ms: 0,
21187 continuation: None,
21188 }),
21189 blocking: 0,
21190 answered: 1,
21191 expected: 1,
21192 clean: false,
21193 progressed: true,
21194 vote_split: false,
21195 reconsideration: Vec::new(),
21196 verdict: None,
21197 };
21198 let state = state_with_round(round);
21199 let body = pr_message(&state, 'A').body;
21200
21201 assert!(body.contains("add retries"), "the task must still be there");
21202 assert!(body.contains("R2-1-1"), "{body}");
21203 assert!(!body.contains("unused import"), "{body}");
21204 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21205 assert!(
21206 !body.contains("not reachable from any caller"),
21207 "the reason it was declined: {body}"
21208 );
21209 }
21210
21211 #[test]
21212 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21213 let round = ReviewRound {
21214 round: 1,
21215 head: "deadbee".to_owned(),
21216 verified_head: None,
21217 verified_at: None,
21218 reviews: vec![ReviewRecord {
21219 attempts: 0,
21220 reviewer: 1,
21221 agent: "alpha".to_owned(),
21222 summary: String::new(),
21223 findings: Vec::new(),
21224 vote: None,
21225 failed: None,
21226 duration_ms: 0,
21227 }],
21228 e2e: Vec::new(),
21229 verify_retried: false,
21230 e2e_deferred: false,
21231 e2e_defer_reason: None,
21232 fix: None,
21233 blocking: 0,
21234 answered: 1,
21235 expected: 1,
21236 clean: true,
21237 progressed: false,
21238 vote_split: false,
21239 reconsideration: Vec::new(),
21240 verdict: None,
21241 };
21242 let state = state_with_round(round);
21243 let body = pr_message(&state, 'A').body;
21244 assert!(!body.contains("Open review findings"), "{body}");
21245 assert!(!body.contains("Declined"), "{body}");
21246 }
21247
21248 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21249 let mut state = RunState::new(
21250 PathBuf::from("/repo"),
21251 "main".to_owned(),
21252 "abc1234".to_owned(),
21253 instruction.to_owned(),
21254 Config::default(),
21255 );
21256 state.config.disk.min_free_bytes = 0;
21258 state.candidates.push(Candidate {
21259 index: 0,
21260 label: 'A',
21261 agent: "alpha".to_owned(),
21262 branch: "magi/x/A".to_owned(),
21263 worktree: PathBuf::from("/wt"),
21264 summary: summary.to_owned(),
21265 stat: String::new(),
21266 files: 1,
21267 commits: 1,
21268 empty: false,
21269 failed: None,
21270 verified_noop: None,
21271 folded: false,
21272 duration_ms: 0,
21273 });
21274 state
21275 }
21276
21277 fn review_state(subjects: &[&str]) -> RunState {
21278 let mut state = state_with_summary(
21279 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21280 "",
21281 );
21282 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21283 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21284 state
21285 }
21286
21287 #[tokio::test]
21290 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21291 ask_test_home();
21292 let tmp = tempfile::tempdir().unwrap();
21293 let repo = tmp.path().join("repo");
21294 std::fs::create_dir_all(&repo).unwrap();
21295 init_repo(&repo);
21296 let origin = tmp.path().join("origin.git");
21297 let g = |dir: &Path, args: &[&str]| {
21298 let out = std::process::Command::new("git")
21299 .args(args)
21300 .current_dir(dir)
21301 .quiet()
21302 .output()
21303 .expect("spawn git");
21304 assert!(
21305 out.status.success(),
21306 "git {args:?}: {}",
21307 String::from_utf8_lossy(&out.stderr)
21308 );
21309 };
21310 g(
21311 tmp.path(),
21312 &[
21313 "clone",
21314 "--bare",
21315 "-q",
21316 repo.to_str().unwrap(),
21317 origin.to_str().unwrap(),
21318 ],
21319 );
21320 g(
21321 &repo,
21322 &["remote", "add", "origin", origin.to_str().unwrap()],
21323 );
21324 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21325
21326 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21328 g(&repo, &["add", "-A"]);
21329 g(
21330 &repo,
21331 &["commit", "-q", "-m", "chore(deps): update a crate"],
21332 );
21333 g(&repo, &["push", "-q", "origin", "main"]);
21334 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21335 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21336 g(&repo, &["add", "-A"]);
21337 g(
21338 &repo,
21339 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21340 );
21341 g(&repo, &["checkout", "-q", "main"]);
21342
21343 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21344 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21345 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21347 assert_eq!(fallback, c1);
21348
21349 let mut state = review_state(&[
21350 "chore(deps): update a crate",
21351 "fix(daemon): apply a chosen action",
21352 ]);
21353 state.repo = repo.clone();
21354 state.base_branch = "main".to_owned();
21355 state.base_commit = c1;
21356 refresh_reviewed_commits(&mut state, "feat/own").await;
21357 assert_eq!(
21358 state.reviewed_commits,
21359 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21360 );
21361 assert_eq!(
21362 review_title(&state).as_deref(),
21363 Some("fix(daemon): apply a chosen action")
21364 );
21365 assert_eq!(
21366 leaked_subjects(&state, "feat/own").await,
21367 Some(vec!["chore(deps): update a crate".to_owned()])
21368 );
21369 state.config.merge.remote = "nowhere".to_owned();
21372 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21373 }
21374
21375 struct SummaryFixture {
21379 _tmp: tempfile::TempDir,
21380 repo: PathBuf,
21381 reply: PathBuf,
21382 runner: Runner,
21383 }
21384
21385 fn summary_fixture() -> SummaryFixture {
21386 ask_test_home();
21387 let tmp = tempfile::tempdir().unwrap();
21388 let repo = tmp.path().join("repo");
21389 std::fs::create_dir_all(&repo).unwrap();
21390 init_repo(&repo);
21391 let origin = tmp.path().join("origin.git");
21392 let g = |args: &[&str]| {
21393 let out = std::process::Command::new("git")
21394 .args(args)
21395 .current_dir(&repo)
21396 .quiet()
21397 .output()
21398 .expect("spawn git");
21399 assert!(
21400 out.status.success(),
21401 "git {args:?}: {}",
21402 String::from_utf8_lossy(&out.stderr)
21403 );
21404 };
21405 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21406 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21407 g(&["push", "-q", "origin", "main"]);
21408 g(&["checkout", "-q", "-b", "feat/own"]);
21409 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21410 g(&["add", "-A"]);
21411 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21412 let reply = tmp.path().join("reply.txt");
21413 std::fs::write(
21414 &reply,
21415 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21416 )
21417 .unwrap();
21418 let mut runner = runner_at(RunStatus::Gating);
21419 let mut state = review_state(&["feat(own): add the own file"]);
21420 state.repo = repo.clone();
21421 state.base_branch = "main".to_owned();
21422 state.candidates[0].branch = "feat/own".to_owned();
21423 state.candidates[0].worktree = repo.clone();
21424 let mut author = spec("alpha");
21425 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21426 author
21427 .env
21428 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21429 state.config.agents = vec![author];
21430 runner.state = state;
21431 SummaryFixture {
21432 _tmp: tmp,
21433 repo,
21434 reply,
21435 runner,
21436 }
21437 }
21438
21439 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21440 let winner = f.runner.state.candidates[0].clone();
21441 f.runner
21442 .guarded_pr_message(&winner, None, true)
21443 .await
21444 .unwrap()
21445 .expect("an English description never parks")
21446 }
21447
21448 fn clean_round_with_minor_findings() -> ReviewRound {
21449 let mut round = passing_round(2, None);
21450 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21451 round.reviews = vec![ReviewRecord {
21452 attempts: 0,
21453 reviewer: 1,
21454 agent: "alpha".into(),
21455 summary: "内部レビュー".into(),
21456 findings: vec![
21457 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21458 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21459 ],
21460 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21461 failed: None,
21462 duration_ms: 0,
21463 }];
21464 round
21465 }
21466
21467 #[tokio::test]
21468 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21469 for review_only in [false, true] {
21470 let mut f = summary_fixture();
21471 if !review_only {
21472 f.runner.state.candidates[0].agent = "alpha".into();
21473 }
21474 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21475 assert!(f.runner.state.reviews[0].clean);
21476 assert!(f.runner.state.reviews[0].fix.is_none());
21477 assert!(f.runner.state.open_findings().is_empty());
21478 assert_eq!(
21479 crate::pr_outcome::required_finding_ids(&f.runner.state),
21480 ["R2-1-1", "R2-1-2"]
21481 );
21482 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21483 let evidence = crate::pr_outcome::summary_evidence(
21484 &f.runner.state,
21485 crate::pr_outcome::FinalHead::At(&head),
21486 );
21487 assert!(
21488 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21489 );
21490 let omitted = posted(&mut f).await;
21492 assert!(omitted.body.contains("No description"));
21493 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21494 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21495 assert!(f.runner.state.change_summary.is_none());
21496 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21497 let described = posted(&mut f).await;
21498 assert!(described.body.contains("improve documentation for callers"));
21499 assert!(described.body.contains("remaining edge case"));
21500 assert!(described.body.contains("## Open review findings"));
21501 assert!(!described.body.contains("未修正"));
21502 }
21503 }
21504
21505 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21506 let mut f = summary_fixture();
21507 if !review_only {
21508 f.runner.state.candidates[0].agent = "alpha".into();
21509 }
21510 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21511 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21512 fixed
21513 .fix
21514 .as_mut()
21515 .unwrap()
21516 .rejected
21517 .push(crate::verdict::Rejection {
21518 id: "R1-1-1".into(),
21519 why: "呼び出されないコードのため辞退".into(),
21520 });
21521 let mut clean = clean_round_with_minor_findings();
21522 clean.verified_head = Some(head.clone());
21523 f.runner.state.reviews = vec![fixed, clean];
21524 ok_gate(&mut f.runner.state, Some(&head));
21525 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21526 f.runner.state.config.agents.push(scripted("beta", reply));
21527 f.runner.state.config.roles.language_judge =
21528 Some(crate::config::AgentChoice::One("beta".into()));
21529 f
21530 }
21531
21532 #[tokio::test]
21533 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21534 for review_only in [false, true] {
21535 let mut f = rewrite_fixture_with_findings(review_only,
21536 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21537 let m = posted(&mut f).await;
21538 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21539 let expected =
21540 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21541 assert!(m.body.contains(&expected));
21542 assert!(m.body.contains("## Review rounds"));
21543 assert!(
21544 m.body
21545 .contains("Gate (`verify.gate`): passed on the final commit")
21546 );
21547 assert!(
21548 m.body
21549 .contains("Tests (`verify.e2e`): passed on the final commit")
21550 );
21551 let c = f.runner.state.change_summary.as_ref().unwrap();
21552 assert!(
21553 m.body
21554 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21555 );
21556 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21557 assert!(crate::pr_outcome::covers_required_findings(
21558 &f.runner.state,
21559 &m.body
21560 ));
21561 assert!(m.body.contains("documentation remains incomplete"));
21562 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21563 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21564 assert!(f.runner.state.github_text.is_none());
21565 }
21566 }
21567
21568 #[tokio::test]
21569 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21570 for review_only in [false, true] {
21571 for reply in [
21572 "TITLE: fix: public description\nA short English description.",
21573 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21574 ] {
21575 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21576 let winner = f.runner.state.candidates[0].clone();
21577 let posted = f
21578 .runner
21579 .guarded_pr_message(&winner, None, true)
21580 .await
21581 .unwrap();
21582 assert!(posted.is_none());
21583 assert!(f.runner.state.parked);
21584 let withheld = f.runner.state.github_text.as_ref().unwrap();
21585 assert!(
21586 withheld.rewrite.is_none(),
21587 "an incomplete rewrite must not be selectable"
21588 );
21589 let q = ask::Questions::open()
21590 .get(withheld.question.as_ref().unwrap())
21591 .unwrap();
21592 assert!(
21593 !q.choices
21594 .contains(&crate::github_text::USE_REWRITE.to_owned())
21595 );
21596 assert!(
21597 q.detail.contains("R1-1-1")
21598 && q.detail.contains("R2-1-1")
21599 && q.detail.contains("R2-1-2")
21600 );
21601 }
21602 }
21603 }
21604
21605 #[tokio::test]
21606 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21607 for review_only in [false, true] {
21608 let mut f = summary_fixture();
21609 if !review_only {
21610 f.runner.state.candidates[0].agent = "alpha".into();
21611 }
21612 f.runner.state.instruction = "内部タスクの説明".into();
21613 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21614 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21615 f.runner.state.reviews = vec![
21616 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21617 round_with_notes(2, Some("二回目の内部修正報告です。")),
21618 passing_round(3, Some(&head)),
21619 ];
21620 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21621 ok_gate(&mut f.runner.state, Some(&head));
21622 let captured = f.repo.join("public-prompt.txt");
21623 let author = &mut f.runner.state.config.agents[0];
21624 author.command = vec![
21625 "sh".into(),
21626 "-c".into(),
21627 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21628 ];
21629 author.env.insert(
21630 "PROMPT_FILE".into(),
21631 captured.to_string_lossy().into_owned(),
21632 );
21633 let m = posted(&mut f).await;
21634 assert!(m.body.contains("Adds `own.txt`"));
21635 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21636 assert!(m.body.contains("R1-1-1"));
21637 assert!(
21638 m.body
21639 .contains("Gate (`verify.gate`): passed on the final commit")
21640 );
21641 assert!(
21642 m.body
21643 .contains("Tests (`verify.e2e`): passed on the final commit")
21644 );
21645 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21646 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21647 let prompt = std::fs::read_to_string(captured).unwrap();
21648 assert!(prompt.contains(&format!(
21649 "git diff {} {head}",
21650 f.runner.state.change_summary.as_ref().unwrap().base
21651 )));
21652 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21653 assert_eq!(
21654 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21655 "二回目の内部修正報告です。"
21656 );
21657 }
21658 }
21659
21660 #[tokio::test]
21661 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21662 for review_only in [false, true] {
21663 let mut f = summary_fixture();
21664 if !review_only {
21665 f.runner.state.candidates[0].agent = "alpha".into();
21666 }
21667 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21668 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21669 std::fs::write(&f.reply, "no usable description").unwrap();
21670 let failed = posted(&mut f).await;
21671 assert!(
21672 failed
21673 .body
21674 .contains("No description of what this change does could be generated")
21675 );
21676 assert!(failed.body.is_ascii());
21677 assert!(f.runner.state.github_text.is_none());
21678 assert!(f.runner.state.change_summary.is_none());
21679 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21680 let evidence = crate::pr_outcome::summary_evidence(
21681 &f.runner.state,
21682 crate::pr_outcome::FinalHead::At(&head),
21683 );
21684 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21685 assert!(
21686 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21687 .exists()
21688 );
21689 f.runner.state =
21690 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21691 std::fs::write(
21692 &f.reply,
21693 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21694 )
21695 .unwrap();
21696 let retried = posted(&mut f).await;
21697 assert!(retried.body.contains("Adds the committed own file."));
21698 assert_eq!(
21699 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21700 "日本語の修正報告"
21701 );
21702 }
21703 }
21704
21705 #[tokio::test]
21706 async fn same_head_verification_and_findings_invalidate_public_prose() {
21707 let mut f = summary_fixture();
21708 posted(&mut f).await;
21709 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21710 let original = f
21711 .runner
21712 .state
21713 .change_summary
21714 .as_ref()
21715 .unwrap()
21716 .evidence
21717 .clone();
21718 ok_gate(&mut f.runner.state, Some(&head));
21719 std::fs::write(
21720 &f.reply,
21721 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21722 )
21723 .unwrap();
21724 let verified = posted(&mut f).await;
21725 assert!(verified.body.contains("now verified"));
21726 assert_ne!(
21727 f.runner.state.change_summary.as_ref().unwrap().evidence,
21728 original
21729 );
21730 let mut round = round_with_notes(1, Some("内部報告"));
21731 round.reviews.push(ReviewRecord {
21732 attempts: 0,
21733 reviewer: 1,
21734 agent: "alpha".into(),
21735 summary: "内部レビュー".into(),
21736 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21737 vote: None,
21738 failed: None,
21739 duration_ms: 0,
21740 });
21741 round
21742 .fix
21743 .as_mut()
21744 .unwrap()
21745 .rejected
21746 .push(crate::verdict::Rejection {
21747 id: "R1-1-1".into(),
21748 why: "呼び出し元から到達できない".into(),
21749 });
21750 f.runner.state.reviews = vec![round];
21751 let omitted = posted(&mut f).await;
21753 assert!(omitted.body.contains("No description"));
21754 assert!(omitted.body.contains("R1-1-1"));
21755 assert!(!omitted.body.contains("到達"));
21756 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21757 let described = posted(&mut f).await;
21758 assert!(described.body.contains("unreachable from callers"));
21759 assert!(described.body.contains("R1-1-1"));
21760 assert!(!described.body.contains("到達"));
21761 let evidence = f
21762 .runner
21763 .state
21764 .change_summary
21765 .as_ref()
21766 .unwrap()
21767 .evidence
21768 .clone();
21769 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21770 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21771 assert!(posted(&mut f).await.body.contains("rationale updated"));
21772 assert_ne!(
21773 f.runner.state.change_summary.as_ref().unwrap().evidence,
21774 evidence
21775 );
21776 f.runner
21778 .state
21779 .change_summary
21780 .as_mut()
21781 .unwrap()
21782 .evidence
21783 .clear();
21784 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21785 assert!(posted(&mut f).await.body.contains("current specification"));
21786 }
21787
21788 #[tokio::test]
21789 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21790 let mut f = summary_fixture();
21791 let m = posted(&mut f).await;
21792 assert_eq!(m.title, "feat(own): add the own file");
21793 assert!(
21794 m.body
21795 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21796 "{}",
21797 m.body
21798 );
21799 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21800 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21801 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21802 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21803 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21804 let kept = f.runner.state.change_summary.as_ref().unwrap();
21805 assert_eq!(
21806 (kept.head.as_str(), kept.base.as_str()),
21807 (head.as_str(), base.as_str())
21808 );
21809 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21810 }
21811
21812 #[tokio::test]
21813 async fn review_only_summary_receives_the_configured_github_language() {
21814 let mut f = summary_fixture();
21815 f.runner.state.config.graph.github_language = "ja".into();
21816 let captured = f.repo.join("summary-prompt.txt");
21817 let author = &mut f.runner.state.config.agents[0];
21818 author.command = vec![
21819 "sh".into(),
21820 "-c".into(),
21821 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21822 ];
21823 author.env.insert(
21824 "PROMPT_FILE".into(),
21825 captured.to_string_lossy().into_owned(),
21826 );
21827 std::fs::write(
21828 &f.reply,
21829 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21830 )
21831 .unwrap();
21832 let m = posted(&mut f).await;
21833 let prompt = std::fs::read_to_string(captured).unwrap();
21834 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21835 assert!(!prompt.contains("in English"));
21836 assert_eq!(m.title, "feat(own): ファイルを追加");
21837 assert!(m.body.contains("own.txt を追加します。"));
21838 }
21839
21840 #[tokio::test]
21841 async fn review_fixes_come_after_the_whole_change_summary() {
21842 let mut f = summary_fixture();
21843 f.runner.state.reviews = vec![
21844 round_with_notes(1, Some("handled the PATH case")),
21845 round_with_notes(2, None),
21846 ];
21847 let m = posted(&mut f).await;
21848 let summary = m.body.find("## Summary").expect("summary");
21849 let fixes = m.body.find("## Review rounds").expect("review rounds");
21850 assert!(summary < fixes, "{}", m.body);
21851 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21852 assert!(m.body.contains("## Final verification"), "{}", m.body);
21853 }
21854
21855 #[tokio::test]
21856 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21857 let mut f = summary_fixture();
21858 let first = posted(&mut f).await;
21859 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21860 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21862 let again = posted(&mut f).await;
21863 assert_eq!((again.title, again.body), (first.title, first.body));
21864
21865 let g = |args: &[&str]| {
21866 let out = std::process::Command::new("git")
21867 .args(args)
21868 .current_dir(&f.repo)
21869 .quiet()
21870 .output()
21871 .expect("spawn git");
21872 assert!(out.status.success(), "git {args:?}");
21873 };
21874 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21875 g(&["add", "-A"]);
21876 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21877 let second = posted(&mut f).await;
21878 let kept = f.runner.state.change_summary.clone().unwrap();
21879 assert_ne!(kept.head, first_head);
21880 assert!(second.body.contains("Not read yet."), "{}", second.body);
21881 assert!(!second.body.contains("so the first summary is read"));
21882 assert_eq!(second.title, "feat(own): other");
21883 }
21884
21885 #[tokio::test]
21886 async fn an_unavailable_summary_is_stated_and_never_invented() {
21887 let mut f = summary_fixture();
21888 std::fs::write(&f.reply, "no title line here").unwrap();
21889 let m = posted(&mut f).await;
21890 assert!(
21891 m.body
21892 .contains("No description of what this change does could be generated"),
21893 "{}",
21894 m.body
21895 );
21896 assert!(m.body.contains("internal reports remain in local records"));
21897 assert!(f.runner.state.change_summary.is_none());
21898 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21899 let winner = f.runner.state.candidates[0].clone();
21901 let preview = f
21902 .runner
21903 .guarded_pr_message(&winner, None, false)
21904 .await
21905 .unwrap()
21906 .unwrap();
21907 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21908 }
21909
21910 #[test]
21911 fn pr_message_review_single_commit_uses_its_subject() {
21912 let state = review_state(&["feat(nats): per-role user"]);
21913 let m = pr_message(&state, 'A');
21914 assert_eq!(m.title, "feat(nats): per-role user");
21915 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21916 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21917 }
21918
21919 #[test]
21920 fn pr_message_review_multi_commit_takes_the_oldest() {
21921 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21922 let m = pr_message(&state, 'A');
21923 assert_eq!(m.title, "feat: the change");
21924 for s in ["feat: the change", "fix: typo", "fix: again"] {
21925 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21926 }
21927 }
21928
21929 #[test]
21930 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21931 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21932 let mut state = review_state(&[first, "fix: later fixup"]);
21933 state.candidates[0].branch = "機能/ログイン".to_owned();
21934 let m = pr_message(&state, 'A');
21935 assert!(
21936 m.title.starts_with("chore: land candidate A of run"),
21937 "{}",
21938 m.title
21939 );
21940 }
21941 }
21942
21943 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21944 BranchFacts {
21945 commits: commits
21946 .iter()
21947 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21948 .collect(),
21949 stat: stat.to_owned(),
21950 }
21951 }
21952
21953 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21954 let mut r = review_round(true, 0, 1, 1, true, true);
21955 r.round = round;
21956 r.fix = notes.map(|n| FixRecord {
21957 agent: "fixer".to_owned(),
21958 addressed: Vec::new(),
21959 rejected: Vec::new(),
21960 notes: n.to_owned(),
21961 committed: true,
21962 failed: None,
21963 duration_ms: 0,
21964 continuation: None,
21965 });
21966 r
21967 }
21968
21969 #[test]
21970 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21971 let state = review_state(&["ignored"]);
21972 let f = facts(
21973 &[
21974 (
21975 "fix(login): resolve PATH on macOS",
21976 "Login shells skip rc files.",
21977 ),
21978 ("fix: address review", ""),
21979 ],
21980 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21981 );
21982 let m = pr_message_with(&state, 'A', Some(&f));
21983 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21984 assert!(
21985 m.body
21986 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
21987 "{}",
21988 m.body
21989 );
21990 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21991 assert!(m.body.contains("## Diff stat"), "{}", m.body);
21992 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21993 for banned in [
21994 "Review the work already",
21995 "no task statement",
21996 "Original task",
21997 ] {
21998 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21999 }
22000 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22001 }
22002
22003 #[test]
22004 fn pr_message_review_truncates_a_huge_first_commit_body() {
22005 let state = review_state(&["ignored"]);
22006 let f = facts(
22007 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22008 "s",
22009 );
22010 let m = pr_message_with(&state, 'A', Some(&f));
22011 assert!(m.body.len() < 30_000, "{}", m.body.len());
22012 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22013 assert!(
22014 m.body.contains("- ... 1 more commit(s)"),
22015 "{}",
22016 m.body.len()
22017 );
22018 assert!(m.body.ends_with("magi:candidate-a\n"));
22019 }
22020
22021 #[test]
22022 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22023 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22024 assert_eq!(m.title, "feat: x");
22025 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22026 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22027 assert!(!m.body.contains("no task statement"), "{}", m.body);
22028 }
22029
22030 #[test]
22031 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22032 let mut state = review_state(&["日本語の件名"]);
22033 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22034 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22035 state.candidates[0].branch = "機能/ログイン".to_owned();
22036 assert!(
22037 pr_message(&state, 'A')
22038 .title
22039 .starts_with("chore: land candidate A")
22040 );
22041 }
22042
22043 #[test]
22044 fn pr_message_review_fixes_survive_a_clean_final_round() {
22045 let mut state = review_state(&["feat: x"]);
22046 state.reviews = vec![
22047 round_with_notes(1, Some("handled the PATH case")),
22048 round_with_notes(2, None),
22049 ];
22050 let body = pr_message(&state, 'A').body;
22051 assert!(
22052 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22053 "{body}"
22054 );
22055 assert!(body.contains("### Round 1"), "{body}");
22056
22057 state.reviews = vec![
22058 round_with_notes(1, Some("first fix")),
22059 round_with_notes(2, Some("")),
22060 round_with_notes(3, Some("second fix")),
22061 round_with_notes(4, None),
22062 ];
22063 let body = pr_message(&state, 'A').body;
22064 assert!(
22065 body.contains("### Round 1") && !body.contains("first fix"),
22066 "{body}"
22067 );
22068 assert!(
22069 body.contains("### Round 3") && !body.contains("second fix"),
22070 "{body}"
22071 );
22072 assert!(body.contains("### Round 2"), "{body}");
22073 }
22074
22075 const FINAL: &str = "abcdef1234567890";
22076
22077 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22078 let mut r = round_with_notes(round, None);
22079 r.verified_head = verified.map(str::to_owned);
22080 r
22081 }
22082
22083 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22084 state.gate = vec![CommandOutcome {
22085 command: "gate".to_owned(),
22086 code: Some(0),
22087 output_tail: String::new(),
22088 duration_ms: 0,
22089 resource_blocked: false,
22090 }];
22091 state.gate_ran = true;
22092 state.gate_head = head.map(str::to_owned);
22093 }
22094
22095 fn body_at(state: &RunState) -> String {
22096 pr_message_raw(
22097 state,
22098 'A',
22099 None,
22100 crate::pr_outcome::FinalHead::At(FINAL),
22101 ChangeNote::NotAsked,
22102 )
22103 .body
22104 }
22105
22106 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22107
22108 #[test]
22109 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22110 let mut state = review_state(&["feat: x"]);
22111 state.reviews = vec![
22112 round_with_notes(1, Some(SANDBOXED)),
22113 passing_round(2, Some(FINAL)),
22114 ];
22115 ok_gate(&mut state, Some(FINAL));
22116 let body = body_at(&state);
22117 assert!(!body.contains("Handled the PATH case."), "{body}");
22118 assert!(!body.contains("could not commit"), "{body}");
22119 assert!(!body.contains("could not run"), "{body}");
22120 assert!(
22121 body.contains("The fix is committed on the branch"),
22122 "{body}"
22123 );
22124 assert!(!body.contains("sandbox"), "{body}");
22125 assert!(
22126 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22127 "{body}"
22128 );
22129 assert!(
22130 body.contains("Gate (`verify.gate`): passed on the final commit"),
22131 "{body}"
22132 );
22133 assert!(
22135 state.reviews[0]
22136 .fix
22137 .as_ref()
22138 .unwrap()
22139 .notes
22140 .contains("could not commit")
22141 );
22142 }
22143
22144 #[test]
22145 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22146 let mut state = review_state(&["feat: x"]);
22147 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22148 r1.fix.as_mut().unwrap().committed = false;
22149 let mut r2 = passing_round(2, Some(FINAL));
22150 r2.e2e[0].code = Some(1);
22151 state.reviews = vec![r1, r2];
22152 let body = body_at(&state);
22153 assert!(!body.contains("could not commit"), "{body}");
22154 assert!(!body.contains("could not run"), "{body}");
22155 assert!(body.contains("This round produced no commit"), "{body}");
22156 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22157 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22158 assert!(!body.contains("passed"), "{body}");
22159 }
22160
22161 #[test]
22162 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22163 let mut state = review_state(&["feat: x"]);
22164 state.reviews = vec![
22165 round_with_notes(1, Some(SANDBOXED)),
22166 passing_round(2, Some("0123456789abcdef")),
22167 ];
22168 ok_gate(&mut state, Some("0123456789abcdef"));
22169 let body = body_at(&state);
22170 assert!(!body.contains("HTTP tests could not run"), "{body}");
22171 assert!(!body.contains("could not commit"), "{body}");
22173 assert!(
22174 body.contains("passed on an earlier commit (`0123456`)"),
22175 "{body}"
22176 );
22177 assert!(!body.contains("passed on the final commit"), "{body}");
22178
22179 ok_gate(&mut state, None);
22181 let body = body_at(&state);
22182 assert!(
22183 body.contains("passed on a commit that was not recorded"),
22184 "{body}"
22185 );
22186 }
22187
22188 #[test]
22189 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22190 let state = state_with_summary(
22191 "add retries to the client",
22192 "TITLE: feat: retries\n\nDid it.",
22193 );
22194 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22195 assert_eq!(m.title, "feat: retries");
22196 assert!(
22197 m.body.contains("<summary>Original task</summary>"),
22198 "{}",
22199 m.body
22200 );
22201 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22202 assert!(
22203 m.body
22204 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22205 "{}",
22206 m.body
22207 );
22208 }
22209
22210 #[test]
22211 fn pr_message_review_bounds_a_long_english_subject() {
22212 let long = format!("feat: {}", "word ".repeat(100));
22213 let m = pr_message(&review_state(&[&long]), 'A');
22214 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22215 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22216 }
22217
22218 #[test]
22219 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22220 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22221 let m = pr_message(&state, 'A');
22222 assert_eq!(m.title, "add retries");
22223 assert!(m.body.contains("<summary>Original task</summary>"));
22224 assert!(!m.body.contains("Commits under review"));
22225 assert_eq!(landing_subject_source(&state), state.instruction);
22226 }
22227
22228 #[test]
22229 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22230 let state = review_state(&["feat: the change", "fix: typo"]);
22231 let source = landing_subject_source(&state);
22232 assert_eq!(land::merge_subject("", &source), "feat: the change");
22233 assert_eq!(
22234 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22235 "feat: the change"
22236 );
22237 assert_eq!(
22239 land::merge_subject("feat: renamed by hand", &source),
22240 "feat: renamed by hand"
22241 );
22242 let blank = review_state(&["日本語"]);
22243 assert!(
22244 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22245 );
22246 }
22247
22248 #[test]
22249 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22250 let state = review_state(&["feat: the change"]);
22251 let source = landing_subject_source(&state);
22252 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22253 assert_eq!(
22254 land::merge_subject(landing_title(&state, old), &source),
22255 "feat: the change"
22256 );
22257 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22258 let task = state_with_summary("add retries", "");
22259 assert_eq!(landing_title(&task, old), old);
22260 }
22261
22262 #[test]
22263 fn pr_message_describes_the_change_not_the_task() {
22264 let state = state_with_summary(
22265 "今回やってほしいこと: results projector を直す",
22266 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22267 );
22268 let m = pr_message(&state, 'A');
22269 assert_eq!(m.title, "fix(web): batch the runs list reads");
22270 assert!(
22271 m.body.starts_with("## Summary\n\n- reads run.json once"),
22272 "{}",
22273 m.body
22274 );
22275 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22276 let task_at = m.body.find("今回やってほしいこと").unwrap();
22277 let details_at = m.body.find("<details>").unwrap();
22278 assert!(
22279 details_at < task_at,
22280 "the task lives inside <details>: {}",
22281 m.body
22282 );
22283 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22284 assert!(m.body.contains("magi:candidate-a"));
22285 }
22286
22287 #[test]
22288 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22289 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22290 let m = pr_message(&state, 'A');
22291 assert_eq!(m.title, "add retries");
22292 assert!(
22293 m.body.contains("## Summary\n\n- did some things"),
22294 "{}",
22295 m.body
22296 );
22297
22298 let none = RunState::new(
22299 PathBuf::from("/repo"),
22300 "main".to_owned(),
22301 "abc1234".to_owned(),
22302 "add retries".to_owned(),
22303 Config::default(),
22304 );
22305 let m = pr_message(&none, 'A');
22306 assert_eq!(m.title, "add retries");
22307 assert!(!m.body.contains("## Summary"), "{}", m.body);
22308 }
22309
22310 #[test]
22311 fn pr_message_refuses_the_candidate_commit_subject() {
22312 for bad in [
22313 "TITLE: magi: candidate A (uncommitted work)",
22314 "TITLE: chore: stuff (uncommitted work)",
22315 "TITLE: ",
22316 ] {
22317 let state = state_with_summary("add retries", bad);
22318 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22319 }
22320 }
22321
22322 #[test]
22323 fn pr_message_bounds_a_very_long_task_and_title() {
22324 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22325 let state = state_with_summary(&long, "- nothing");
22326 let m = pr_message(&state, 'A');
22327 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22328 assert!(!m.title.contains('\n'));
22329
22330 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22331 let m = pr_message(&state, 'A');
22332 assert!(m.title.starts_with("feat: "));
22333 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22334 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22335 }
22336
22337 fn long_title_of(instruction: &str) -> String {
22338 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22339 }
22340
22341 #[test]
22342 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22343 let first = "Make the landing path keep a readable title for long tasks";
22344 let line = format!(
22345 "{first}. {}",
22346 "And then keep going with more words ".repeat(20)
22347 );
22348 let t = long_title_of(&line);
22349 assert_eq!(t, first);
22350 assert!(!t.starts_with("chore: land"));
22351 }
22352
22353 #[test]
22354 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22355 let line = "word ".repeat(200);
22356 let t = long_title_of(&line);
22357 assert!(t.ends_with("word..."), "{t}");
22358 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22359 }
22360
22361 #[test]
22362 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22363 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22364 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22365 }
22366 }
22367
22368 #[test]
22369 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22370 let mut state = state_with_summary("再試行を追加する", "- nothing");
22371 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22372 state.config.graph.github_language = "ja".into();
22373 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22374 }
22375
22376 #[test]
22377 fn pr_message_title_limit_is_exact() {
22378 let at = "a".repeat(PR_TITLE_MAX);
22379 assert_eq!(long_title_of(&at), at);
22380 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22381 assert!(over.ends_with("..."), "{over}");
22382 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22383 }
22384
22385 #[test]
22386 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22387 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22388 let t = long_title_of(&line);
22389 assert!(t.ends_with("..."), "{t}");
22390 assert!(t.is_ascii(), "{t}");
22391 }
22392
22393 #[test]
22394 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22395 let line = format!(
22396 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22397 "plus more filler words ".repeat(20)
22398 );
22399 let t = long_title_of(&line);
22400 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22401 }
22402
22403 #[test]
22404 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22405 let line = format!("\n\n# {}", "title words ".repeat(40));
22406 let state = state_with_summary(&line, "- nothing");
22407 let m = pr_message(&state, 'A');
22408 assert!(m.title.starts_with("title words"), "{}", m.title);
22409 assert_eq!(
22410 land::merge_subject(&m.title, &landing_subject_source(&state)),
22411 m.title
22412 );
22413 assert_eq!(
22415 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22416 "feat: renamed by hand"
22417 );
22418 }
22419
22420 #[tokio::test]
22421 async fn github_text_rewrite_is_bounded_and_falls_back() {
22422 crate::run::pin_test_home();
22423 for (reply, accepted) in [
22424 (
22425 "TITLE: fix: retries\nAdd retries for failed requests.",
22426 true,
22427 ),
22428 (
22429 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22430 false,
22431 ),
22432 (
22433 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22434 true,
22435 ),
22436 (
22437 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22438 true,
22439 ),
22440 (
22441 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22442 true,
22443 ),
22444 (
22445 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22446 false,
22447 ),
22448 ("TITLE: fix: retries\n", false),
22449 ] {
22450 let dir = tempfile::tempdir().unwrap();
22451 let mut runner = runner_at(RunStatus::Gating);
22452 runner.state = state_with_summary(
22453 "add retries",
22454 "TITLE: fix: retries\n日本語の説明を書きます。",
22455 );
22456 runner.state.repo = dir.path().to_owned();
22457 runner.state.candidates[0].worktree = dir.path().to_owned();
22458 let mut author = spec("alpha");
22459 author.command = vec![
22460 "sh".into(),
22461 "-c".into(),
22462 "printf '%s' \"$REWRITE_REPLY\"".into(),
22463 ];
22464 author.env.insert("REWRITE_REPLY".into(), reply.into());
22465 runner.state.config.agents = vec![author];
22466 seed_public_summary(&mut runner);
22467 let winner = runner.state.candidates[0].clone();
22468 let first = runner
22469 .guarded_pr_message(&winner, None, true)
22470 .await
22471 .unwrap();
22472 let message = if accepted {
22473 first.expect("a passing rewrite never asks")
22474 } else {
22475 assert!(first.is_none());
22477 assert!(runner.state.parked);
22478 let store = ask::Questions::open();
22479 let asked: Vec<_> = store
22480 .list()
22481 .into_iter()
22482 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22483 .collect();
22484 assert_eq!(asked.len(), 1);
22485 assert!(!asked[0].detail.contains("secret"));
22486 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22487 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22488 assert_eq!(
22489 asked[0]
22490 .choices
22491 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22492 rewrite.is_some()
22493 );
22494 store
22497 .update(&asked[0].id, |q| {
22498 q.abandon("test");
22499 Ok(())
22500 })
22501 .unwrap();
22502 let again = runner
22503 .guarded_pr_message(&winner, None, true)
22504 .await
22505 .unwrap()
22506 .expect("an abandoned question falls back");
22507 let n = store
22508 .list()
22509 .into_iter()
22510 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22511 .count();
22512 assert_eq!(n, 1, "asked once per rejected text");
22513 again
22514 };
22515 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22516 assert!(!message.body.contains("secret"));
22517 assert!(!message.body.contains("prefix"));
22518 assert!(!message.title.contains("secret"));
22519 assert_eq!(
22520 message.body.contains("Add retries for failed requests."),
22521 accepted
22522 );
22523 assert_eq!(
22524 runner.state.seats["github-text-rewrite"].turns, 1,
22525 "only one rewrite invocation"
22526 );
22527 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22528 if accepted {
22529 assert!(runner.state.github_text.is_none());
22530 assert!(!runner.state.parked);
22531 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22532 assert!(message.body.contains("## Final verification"));
22533 assert!(message.body.contains("Evidence: final diff"));
22534 assert!(
22535 message
22536 .body
22537 .contains(&format!("magi:run/{}", runner.state.id))
22538 );
22539 }
22540 if !accepted {
22541 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22542 assert!(
22543 message
22544 .body
22545 .contains(&format!("magi:run/{}", runner.state.id))
22546 );
22547 }
22548 }
22549 }
22550
22551 fn scripted(id: &str, reply: &str) -> AgentSpec {
22552 let mut a = spec(id);
22553 a.command = vec![
22554 "sh".into(),
22555 "-c".into(),
22556 "printf '%s' \"$REWRITE_REPLY\"".into(),
22557 ];
22558 a.env.insert("REWRITE_REPLY".into(), reply.into());
22559 a
22560 }
22561
22562 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22563 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22564
22565 fn seed_public_summary(runner: &mut Runner) {
22567 let repo = &runner.state.repo;
22568 let g = |args: &[&str]| {
22569 let out = std::process::Command::new("git")
22570 .args(args)
22571 .current_dir(repo)
22572 .quiet()
22573 .output()
22574 .unwrap();
22575 assert!(out.status.success(), "git {args:?}");
22576 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22577 };
22578 g(&["init", "-q", "-b", "main"]);
22579 g(&[
22580 "-c",
22581 "user.name=Test",
22582 "-c",
22583 "user.email=test@example.invalid",
22584 "commit",
22585 "-q",
22586 "--allow-empty",
22587 "-m",
22588 "feat: fixture",
22589 ]);
22590 let head = g(&["rev-parse", "HEAD"]);
22591 g(&["update-ref", "refs/remotes/origin/main", &head]);
22592 let evidence = crate::pr_outcome::summary_evidence(
22593 &runner.state,
22594 crate::pr_outcome::FinalHead::At(&head),
22595 );
22596 let generated = &runner.state.candidates[0].summary;
22597 runner.state.change_summary = Some(crate::run::ChangeSummary {
22598 base: head.clone(),
22599 head,
22600 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22601 title: summary_title(generated).unwrap(),
22602 body: summary_without_title(generated),
22603 });
22604 }
22605
22606 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22607 let mut runner = runner_at(RunStatus::Gating);
22608 runner.state = state_with_summary("add retries", JA_SUMMARY);
22609 if review_only {
22610 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22611 }
22612 runner.state.repo = dir.to_owned();
22613 runner.state.candidates[0].worktree = dir.to_owned();
22614 runner.state.config.agents = agents;
22615 seed_public_summary(&mut runner);
22616 runner
22617 }
22618
22619 fn gate_questions(runner: &Runner) -> usize {
22620 ask::Questions::open()
22621 .list()
22622 .into_iter()
22623 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22624 .count()
22625 }
22626
22627 #[tokio::test]
22628 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22629 crate::run::pin_test_home();
22630 let dir = tempfile::tempdir().unwrap();
22631 let mut runner = rewrite_runner(
22632 dir.path(),
22633 true,
22634 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22635 );
22636 runner.state.config.roles.language_judge =
22637 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22638 let winner = runner.state.candidates[0].clone();
22639 let msg = runner
22640 .guarded_pr_message(&winner, None, true)
22641 .await
22642 .unwrap()
22643 .expect("an English rewrite posts without asking");
22644 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22645 assert!(msg.body.contains("<details>kept</details>"));
22646 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22647 assert_eq!(gate_questions(&runner), 0);
22648 assert!(runner.state.github_text.is_none());
22649 assert!(!runner.state.seats.contains_key("impl-A"));
22650 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22651 }
22652
22653 #[tokio::test]
22654 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22655 crate::run::pin_test_home();
22656 let dir = tempfile::tempdir().unwrap();
22657 let mut runner = rewrite_runner(
22658 dir.path(),
22659 false,
22660 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22661 );
22662 runner.state.config.roles.language_judge =
22663 Some(crate::config::AgentChoice::One("beta".into()));
22664 runner.state.seats.insert(
22665 "impl-A".into(),
22666 SeatState::new("impl-A", "alpha", runner.state.seed),
22667 );
22668 let winner = runner.state.candidates[0].clone();
22669 let msg = runner
22670 .guarded_pr_message(&winner, None, true)
22671 .await
22672 .unwrap()
22673 .expect("rewritten");
22674 assert!(msg.body.contains("<details>kept</details>"));
22675 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22676 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22677 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22678 }
22679
22680 #[tokio::test]
22681 async fn rewrite_reuses_a_resumable_implementer_session() {
22682 crate::run::pin_test_home();
22683 let dir = tempfile::tempdir().unwrap();
22684 let mut runner = rewrite_runner(
22685 dir.path(),
22686 false,
22687 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22688 );
22689 runner.state.config.roles.language_judge =
22690 Some(crate::config::AgentChoice::One("beta".into()));
22691 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22692 seat.turns = 1;
22693 runner.state.seats.insert("impl-A".into(), seat);
22694 let winner = runner.state.candidates[0].clone();
22695 runner
22696 .guarded_pr_message(&winner, None, true)
22697 .await
22698 .unwrap()
22699 .expect("rewritten");
22700 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22701 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22702 }
22703
22704 #[tokio::test]
22705 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22706 crate::run::pin_test_home();
22709 let dir = tempfile::tempdir().unwrap();
22710 let mut exhausted = spec("alpha");
22711 exhausted.command = vec!["false".into()];
22712 let mut runner = rewrite_runner(
22713 dir.path(),
22714 false,
22715 vec![exhausted, scripted("beta", EN_REWRITE)],
22716 );
22717 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22719 "alpha".into(),
22720 "beta".into(),
22721 ]));
22722 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22723 seat.turns = 1;
22724 runner.state.seats.insert("impl-A".into(), seat);
22725 let winner = runner.state.candidates[0].clone();
22726 let msg = runner
22727 .guarded_pr_message(&winner, None, true)
22728 .await
22729 .unwrap()
22730 .expect("the role rewrites after the implementer fails");
22731 assert!(msg.body.contains("`src/net.rs`"));
22732 assert_eq!(
22733 runner.state.seats["impl-A"].turns, 2,
22734 "the implementer was tried once"
22735 );
22736 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22737 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22738 }
22739
22740 #[tokio::test]
22741 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22742 crate::run::pin_test_home();
22743 let dir = tempfile::tempdir().unwrap();
22744 let mut broken = spec("bad");
22745 broken.command = vec!["false".into()];
22746 let mut runner = rewrite_runner(
22747 dir.path(),
22748 true,
22749 vec![
22750 broken,
22751 scripted("noformat", "Add retries."),
22752 scripted("good", EN_REWRITE),
22753 ],
22754 );
22755 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22756 "bad".into(),
22757 "noformat".into(),
22758 "good".into(),
22759 ]));
22760 let winner = runner.state.candidates[0].clone();
22761 let msg = runner
22762 .guarded_pr_message(&winner, None, true)
22763 .await
22764 .unwrap()
22765 .expect("the third agent rewrites");
22766 assert!(msg.body.contains("`src/net.rs`"));
22767 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22768 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22769 }
22770
22771 #[tokio::test]
22772 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22773 crate::run::pin_test_home();
22774 for (agents, role, saved) in [
22776 (
22777 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22778 Some("beta"),
22779 true,
22780 ),
22781 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22782 (Vec::new(), None, false),
22783 ] {
22784 let dir = tempfile::tempdir().unwrap();
22785 let mut runner = rewrite_runner(dir.path(), true, agents);
22786 runner.state.config.roles.language_judge =
22787 role.map(|r| crate::config::AgentChoice::One(r.into()));
22788 let winner = runner.state.candidates[0].clone();
22789 let first = runner
22790 .guarded_pr_message(&winner, None, true)
22791 .await
22792 .unwrap();
22793 assert!(first.is_none(), "withheld text parks the run");
22794 assert!(runner.state.parked);
22795 assert_eq!(gate_questions(&runner), 1);
22796 let g = runner.state.github_text.as_ref().unwrap();
22797 assert_eq!(g.rewrite.is_some(), saved);
22798 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22799 }
22800 }
22801
22802 #[tokio::test]
22803 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22804 crate::run::pin_test_home();
22805 let dir = tempfile::tempdir().unwrap();
22806 let mut runner = runner_at(RunStatus::Gating);
22807 runner.state =
22808 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22809 runner.state.repo = dir.path().to_owned();
22810 runner.state.candidates[0].worktree = dir.path().to_owned();
22811 let mut author = spec("alpha");
22812 author.command = vec![
22813 "sh".into(),
22814 "-c".into(),
22815 "printf '%s' \"$REWRITE_REPLY\"".into(),
22816 ];
22817 author.env.insert(
22818 "REWRITE_REPLY".into(),
22819 format!(
22820 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22821 "English sentence. ".repeat(600)
22822 ),
22823 );
22824 runner.state.config.agents = vec![author];
22825 seed_public_summary(&mut runner);
22826 let winner = runner.state.candidates[0].clone();
22827 assert!(
22828 runner
22829 .guarded_pr_message(&winner, None, true)
22830 .await
22831 .unwrap()
22832 .is_none()
22833 );
22834 let record = runner.state.github_text.clone().unwrap();
22835 let rewrite = record.rewrite.clone().unwrap();
22836 assert_eq!(rewrite.failed, ["body-language"]);
22837 assert!(
22838 rewrite
22839 .body
22840 .ends_with(&format!("magi:run/{}", runner.state.id))
22841 );
22842 let store = ask::Questions::open();
22843 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22844 assert!(
22845 q.choices
22846 .contains(&crate::github_text::USE_REWRITE.to_owned())
22847 );
22848 for label in [
22849 "ORIGINAL",
22850 "FALLBACK",
22851 "REWRITE",
22852 "Rules still failed: body-language",
22853 "truncated",
22854 ] {
22855 assert!(q.detail.contains(label), "{label}");
22856 }
22857 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22858 &runner.state,
22859 &crate::github_text::artifact_name(&record.fingerprint),
22860 ))
22861 .unwrap();
22862 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22863 runner.state =
22865 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22866 store
22867 .update(&q.id, |q| {
22868 q.say("まだ日本語")?;
22869 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22870 })
22871 .unwrap();
22872 assert!(
22873 runner
22874 .guarded_pr_message(&winner, None, true)
22875 .await
22876 .unwrap()
22877 .is_none()
22878 );
22879 let retry = runner.state.github_text.as_ref().unwrap();
22880 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22881 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22882 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22883 assert!(
22884 q.posting_comparison
22885 .as_ref()
22886 .unwrap()
22887 .matches_detail(&q.detail)
22888 );
22889 store
22890 .update(&q.id, |q| {
22891 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22892 })
22893 .unwrap();
22894 let message = runner
22895 .guarded_pr_message(&winner, None, true)
22896 .await
22897 .unwrap()
22898 .unwrap();
22899 assert_eq!(message.title, rewrite.title);
22900 assert_eq!(message.body, rewrite.body);
22901 assert_eq!(
22902 runner.state.github_text.as_ref().unwrap().choice,
22903 Some(GithubTextChoice::Rewrite)
22904 );
22905 let again = runner
22906 .guarded_pr_message(&winner, None, true)
22907 .await
22908 .unwrap()
22909 .unwrap();
22910 assert_eq!(again.title, message.title);
22911 assert_eq!(again.body, message.body);
22912 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22913 assert!(
22914 !runner
22915 .state
22916 .events
22917 .iter()
22918 .any(|e| e.message.contains(&rewrite.title)
22919 || e.message.contains("English sentence."))
22920 );
22921 }
22922
22923 #[tokio::test]
22924 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22925 crate::run::pin_test_home();
22926 let dir = tempfile::tempdir().unwrap();
22927 let mut runner = runner_at(RunStatus::Gating);
22928 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22929 runner.state.repo = dir.path().to_owned();
22930 runner.state.candidates[0].worktree = dir.path().to_owned();
22931 seed_public_summary(&mut runner);
22932 let winner = runner.state.candidates[0].clone();
22933 assert!(
22934 runner
22935 .guarded_pr_message(&winner, None, true)
22936 .await
22937 .unwrap()
22938 .is_none()
22939 );
22940 let record = runner.state.github_text.as_ref().unwrap();
22941 assert!(record.rewrite.is_none());
22942 let mut old = serde_json::to_value(record).unwrap();
22944 old.as_object_mut().unwrap().remove("rewrite");
22945 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22946 assert!(old.rewrite.is_none());
22947 let id = record.question.clone().unwrap();
22948 ask::Questions::open()
22949 .update(&id, |q| {
22950 q.choices.push(crate::github_text::USE_REWRITE.into());
22951 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22952 })
22953 .unwrap();
22954 let message = runner
22955 .guarded_pr_message(&winner, None, true)
22956 .await
22957 .unwrap()
22958 .unwrap();
22959 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22960 assert_eq!(
22961 runner.state.github_text.as_ref().unwrap().choice,
22962 Some(GithubTextChoice::Fallback)
22963 );
22964 }
22965
22966 #[tokio::test]
22967 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22968 crate::run::pin_test_home();
22969 let dir = tempfile::tempdir().unwrap();
22970 let mut runner = runner_at(RunStatus::Gating);
22971 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22972 runner.state.repo = dir.path().to_owned();
22973 runner.state.candidates[0].worktree = dir.path().to_owned();
22974 seed_public_summary(&mut runner);
22975 let winner = runner.state.candidates[0].clone();
22976 assert!(
22977 runner
22978 .guarded_pr_message(&winner, None, true)
22979 .await
22980 .unwrap()
22981 .is_none()
22982 );
22983 let store = ask::Questions::open();
22984 let find = |run: &str| {
22985 store
22986 .list()
22987 .into_iter()
22988 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22989 .unwrap()
22990 };
22991 let run = runner.state.id.clone();
22992 let q = find(&run);
22993 let fp = runner
22994 .state
22995 .github_text
22996 .as_ref()
22997 .unwrap()
22998 .fingerprint
22999 .clone();
23000 let name = crate::github_text::artifact_name(&fp);
23001 let path = crate::run::artifact_path(&runner.state, &name);
23002 assert!(q.detail.contains(&path.display().to_string()));
23003 assert!(q.detail.contains("title-language: title"));
23004 let saved = std::fs::read_to_string(&path).expect("artifact written");
23005 assert!(saved.contains("日本語のタイトル"));
23006 assert!(q.cwd.is_none());
23007 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23008 let say = |id: &str, text: &str, choice: &str| {
23010 store
23011 .update(id, |q| {
23012 q.thread.push(ask::Turn {
23013 who: ask::Who::Operator,
23014 body: text.to_owned(),
23015 at: jiff::Timestamp::now(),
23016 note: None,
23017 });
23018 q.answer(ask::Answer::Choice(choice.to_owned()))
23019 })
23020 .unwrap();
23021 };
23022 say(&q.id, "まだ日本語", "use my text");
23023 assert!(
23024 runner
23025 .guarded_pr_message(&winner, None, true)
23026 .await
23027 .unwrap()
23028 .is_none()
23029 );
23030 let second = find(&run);
23031 assert_ne!(second.id, q.id);
23032 assert!(second.detail.contains("did not pass"));
23033 assert!(second.detail.contains("日本語のタイトル"));
23036 assert!(!second.detail.contains("まだ日本語"));
23037 assert!(second.detail.contains(&path.display().to_string()));
23038 say(&second.id, "fix: retry failed requests", "use my text");
23039 let message = runner
23040 .guarded_pr_message(&winner, None, true)
23041 .await
23042 .unwrap()
23043 .expect("a vetted replacement is posted");
23044 assert_eq!(message.title, "fix: retry failed requests");
23045 assert_eq!(
23046 runner
23047 .state
23048 .github_text
23049 .as_ref()
23050 .unwrap()
23051 .chosen_title
23052 .as_deref(),
23053 Some("fix: retry failed requests")
23054 );
23055 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23056 let again = runner
23058 .guarded_pr_message(&winner, None, true)
23059 .await
23060 .unwrap()
23061 .unwrap();
23062 assert_eq!(again.title, message.title);
23063 assert_eq!(
23064 store
23065 .list()
23066 .into_iter()
23067 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23068 .count(),
23069 2
23070 );
23071 }
23072
23073 #[test]
23074 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23075 let mut state = state_with_summary(
23079 "add retries",
23080 "TITLE: fix(web): batch reads\n- reads run.json once",
23081 );
23082 state.config.graph.language = "ja".to_owned();
23083 let m = pr_message(&state, 'A');
23084 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23085 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23086
23087 let task = "今回やってほしいこと: results projector を直す";
23090 let mut state = state_with_summary(task, "- no title line");
23091 state.config.graph.language = "ja".to_owned();
23092 let m = pr_message(&state, 'A');
23093 assert_eq!(
23094 m.title,
23095 format!("chore: land candidate A of run {}", state.id)
23096 );
23097 assert!(
23098 m.body.contains(&format!(
23099 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23100 )),
23101 "{}",
23102 m.body
23103 );
23104 }
23105
23106 #[test]
23107 fn pr_message_scrubs_home_paths_and_addresses() {
23108 let state = state_with_summary(
23109 "fix it in /Users/someone/src/x",
23110 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23111 );
23112 let m = pr_message(&state, 'A');
23113 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23114 assert!(!m.body.contains(leak), "{}", m.body);
23115 }
23116 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23117 }
23118
23119 #[test]
23120 fn pr_message_survives_a_task_that_closes_details() {
23121 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23122 let m = pr_message(&state, 'A');
23123 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23124 }
23125
23126 #[test]
23127 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23128 let cmd = manual_merge_command(
23129 MergeStyle::Squash,
23130 Path::new("/repo"),
23131 "b",
23132 "fix: \"quoted\" $(x) `y`\n\nbody",
23133 );
23134 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23135 }
23136
23137 #[test]
23138 fn manual_merge_command_matches_the_configured_style() {
23139 let repo = Path::new("/repo");
23140 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23141
23142 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23143 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23144
23145 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23146 assert_eq!(
23147 squash,
23148 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23149 \"Merge magi run 0832 (candidate A)\""
23150 );
23151
23152 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23153 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23154 }
23155
23156 #[test]
23157 fn a_nudge_gets_a_quarter_of_the_budget() {
23158 assert_eq!(retry_budget(secs(1200), true), secs(300));
23160 assert_eq!(retry_budget(secs(3600), true), secs(900));
23161 }
23162
23163 #[test]
23164 fn a_resent_prompt_keeps_the_whole_budget() {
23165 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23168 assert_eq!(retry_budget(secs(60), false), secs(60));
23169 }
23170
23171 #[test]
23172 fn the_floor_never_exceeds_the_original_budget() {
23173 assert_eq!(retry_budget(secs(60), true), secs(60));
23177 assert_eq!(retry_budget(secs(480), true), secs(120));
23178 assert_eq!(retry_budget(secs(0), true), secs(0));
23179 }
23180
23181 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23182 agent::CommandEvidence {
23183 id: "item1".to_owned(),
23184 description: "cargo test".to_owned(),
23185 exit_code,
23186 result_summary: String::new(),
23187 source: "codex".to_owned(),
23188 }
23189 }
23190
23191 #[test]
23192 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23193 assert!(!has_unconfirmed_command(&[]));
23197 }
23198
23199 #[test]
23200 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23201 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23205 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23206 assert!(!has_unconfirmed_command(&[
23207 evidence(Some(0)),
23208 evidence(Some(101))
23209 ]));
23210 }
23211
23212 #[test]
23213 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23214 assert!(has_unconfirmed_command(&[
23215 evidence(Some(0)),
23216 evidence(None)
23217 ]));
23218 }
23219
23220 #[test]
23221 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23222 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23223 assert_eq!(
23224 verified_noop_claim(true, &[], text).as_deref(),
23225 Some("already fixed by b32cfc4, on main.")
23226 );
23227 }
23228
23229 #[test]
23230 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23231 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23234 assert!(verified_noop_claim(false, &[], text).is_none());
23235 }
23236
23237 #[test]
23238 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23239 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23240 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23241 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23243 }
23244
23245 #[test]
23246 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23247 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23248 }
23249
23250 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23253 runner.state.candidates = shape
23254 .iter()
23255 .enumerate()
23256 .map(|(i, &(empty, verified))| Candidate {
23257 index: i,
23258 label: (b'A' + i as u8) as char,
23259 agent: "sonnet".to_owned(),
23260 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23261 worktree: PathBuf::from(format!("/wt/{i}")),
23262 summary: String::new(),
23263 stat: String::new(),
23264 files: 0,
23265 commits: 0,
23266 empty,
23267 failed: None,
23268 verified_noop: verified.map(str::to_owned),
23269 duration_ms: 0,
23270 folded: false,
23271 })
23272 .collect();
23273 }
23274
23275 #[test]
23276 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23277 ask_test_home();
23278 let mut runner = runner_at(RunStatus::Implementing);
23279 set_candidates(
23280 &mut runner,
23281 &[
23282 (true, Some("already on main at b32cfc4")),
23283 (true, Some("same fix, see the existing test")),
23284 ],
23285 );
23286
23287 runner
23288 .after_implement()
23289 .expect("a verified no-op is not an error");
23290
23291 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23292 }
23293
23294 #[test]
23295 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23296 ask_test_home();
23297 let mut runner = runner_at(RunStatus::Implementing);
23298 set_candidates(
23302 &mut runner,
23303 &[(true, Some("already on main at b32cfc4")), (true, None)],
23304 );
23305
23306 let err = runner
23307 .after_implement()
23308 .expect_err("an unverified empty candidate must still fail the run");
23309
23310 assert!(
23311 err.to_string().contains("no candidate produced a change"),
23312 "{err}"
23313 );
23314 assert_eq!(runner.state.status, RunStatus::Failed);
23315 }
23316
23317 #[test]
23318 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23319 ask_test_home();
23320 let mut runner = runner_at(RunStatus::Implementing);
23321 set_candidates(&mut runner, &[(true, None), (true, None)]);
23322
23323 let err = runner
23324 .after_implement()
23325 .expect_err("no candidate declared anything; this is an ordinary failure");
23326
23327 assert!(
23328 err.to_string().contains("no candidate produced a change"),
23329 "{err}"
23330 );
23331 assert_eq!(runner.state.status, RunStatus::Failed);
23332 }
23333}
23334
23335#[derive(Debug)]
23338struct GraftRestoreFailed(String);
23339
23340impl std::fmt::Display for GraftRestoreFailed {
23341 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23342 write!(
23343 f,
23344 "could not restore the winner to its pre-graft commit: {}",
23345 self.0
23346 )
23347 }
23348}
23349
23350impl std::error::Error for GraftRestoreFailed {}
23351
23352#[cfg(test)]
23356mod resource_wait_tests {
23357 use super::*;
23358 use std::collections::BTreeMap;
23359
23360 fn sh(id: &str, script: &str) -> AgentSpec {
23361 AgentSpec {
23362 id: id.to_owned(),
23363 kind: crate::config::AgentKind::Command,
23364 model: None,
23365 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23366 extra_args: Vec::new(),
23367 env: BTreeMap::new(),
23368 prompt_delivery: None,
23369 max_turn_secs: None,
23370 }
23371 }
23372
23373 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23374 SeatJob {
23375 seat: SeatState::new("impl-A", &spec.id, 1),
23376 spec,
23377 cwd: dir.to_path_buf(),
23378 prompt: "do it".to_owned(),
23379 timeout,
23380 allow_write: true,
23381 sessions: false,
23382 artifacts: dir.join("art"),
23383 stem: "impl-A".to_owned(),
23384 handover: None,
23385 }
23386 }
23387
23388 fn state(resource_wait_secs: u64) -> RunState {
23389 let mut config = Config::default();
23390 config.graph.timeout_resource_wait = resource_wait_secs;
23391 RunState::new(
23392 PathBuf::from("/repo"),
23393 "main".to_owned(),
23394 "abc1234".to_owned(),
23395 "task".to_owned(),
23396 config,
23397 )
23398 }
23399
23400 fn hold(cache: &Path) -> crate::cache::Guard {
23401 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23402 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23403 crate::cache::AcquireOutcome::Acquired(g) => g,
23404 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23405 }
23406 }
23407
23408 async fn run(
23409 jobs: Vec<SeatJob>,
23410 sem: Arc<Semaphore>,
23411 cache: Option<&Path>,
23412 st: &mut RunState,
23413 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23414 let prompts = Prompts::default();
23415 let ctx = WaveCtx {
23416 run: "r",
23417 node: "implement",
23418 prompts: &prompts,
23419 cache,
23420 round: None,
23421 carry_seats: false,
23422 pauses: None,
23423 };
23424 wave(jobs, sem, &ctx, st, 0).await
23425 }
23426
23427 #[tokio::test]
23428 async fn verification_cache_wait_preserves_the_command_budget() {
23429 crate::run::pin_test_home();
23430 let tmp = tempfile::tempdir().unwrap();
23431 let cache = tmp.path().join("target");
23432 let guard = hold(&cache);
23433 let release = tokio::spawn(async move {
23434 tokio::time::sleep(Duration::from_millis(100)).await;
23435 guard.release();
23436 });
23437 let mut st = state(10);
23438 let budget = Duration::from_millis(50);
23439 let wait = repository_wait(&st, None);
23440 let (outcomes, _) = with_cache_lease(
23441 &mut st,
23442 Some(&cache),
23443 "gate",
23444 "gate",
23445 tmp.path(),
23446 "abc1234",
23447 budget,
23448 "test gate",
23449 &wait,
23450 |_, available| async move {
23451 assert_eq!(
23452 available, budget,
23453 "contention must not shorten verification"
23454 );
23455 (Vec::new(), false, Vec::new())
23456 },
23457 )
23458 .await
23459 .unwrap();
23460 release.await.unwrap();
23461 assert!(outcomes.is_empty(), "{outcomes:?}");
23462 assert!(
23463 st.events
23464 .iter()
23465 .any(|event| event.message.contains("waiting for"))
23466 );
23467 assert!(st.resource_wait.is_none());
23468 hold(&cache).release();
23470 }
23471
23472 #[tokio::test]
23473 async fn verification_cache_wait_expires_without_starting_commands() {
23474 crate::run::pin_test_home();
23475 let tmp = tempfile::tempdir().unwrap();
23476 let cache = tmp.path().join("target");
23477 let _held = hold(&cache);
23478 let mut st = state(0);
23479 let wait = repository_wait(&st, None);
23480 let (outcomes, _) = with_cache_lease(
23481 &mut st,
23482 Some(&cache),
23483 "gate",
23484 "gate",
23485 tmp.path(),
23486 "abc1234",
23487 Duration::from_secs(60),
23488 "test gate",
23489 &wait,
23490 |_, _| async { panic!("commands must not start without the cache lease") },
23491 )
23492 .await
23493 .unwrap();
23494 assert_eq!(outcomes.len(), 1);
23495 assert!(outcomes[0].resource_blocked);
23496 assert!(st.resource_wait.is_none());
23497 assert!(matches!(
23498 st.resource_block.unwrap().cause,
23499 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23500 ));
23501 }
23502
23503 #[tokio::test]
23504 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23505 crate::run::pin_test_home();
23506 let tmp = tempfile::tempdir().unwrap();
23507 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23508 let cache = tmp.path().join("cache");
23509 let mut state = state(21600);
23510 let wait = repository_wait(&state, None);
23511 let (outcomes, _) = with_cache_lease(
23512 &mut state,
23513 Some(&cache),
23514 "gate",
23515 "gate",
23516 tmp.path(),
23517 "head",
23518 Duration::from_secs(60),
23519 "gate",
23520 &wait,
23521 |_, _| async { panic!("freshness failure must not start verification") },
23522 )
23523 .await
23524 .unwrap();
23525 assert_eq!(outcomes.len(), 1);
23526 assert!(outcomes[0].resource_blocked);
23527 let block = state.resource_block.unwrap();
23528 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23529 assert!(!block.description().contains("timed out"));
23530 }
23531
23532 #[tokio::test]
23533 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23534 crate::run::pin_test_home();
23535 let tmp = tempfile::tempdir().unwrap();
23536 let cache = tmp.path().join("cache");
23537 let _held = hold(&cache);
23538 let mut state = state(21600);
23539 let pause = Pause::new();
23540 let interrupt = Pause::new();
23541 let cause = ParkCause::Operator;
23542 pause.park_for(cause.clone());
23543 let prompts = Prompts::default();
23544 let ctx = WaveCtx {
23545 run: "test",
23546 node: "fix",
23547 prompts: &prompts,
23548 cache: Some(&cache),
23549 round: None,
23550 carry_seats: false,
23551 pauses: Some((&pause, &interrupt)),
23552 };
23553 let result = wave(
23554 vec![job(
23555 sh("writer", "exit 99"),
23556 tmp.path(),
23557 Duration::from_secs(1),
23558 )],
23559 Arc::new(Semaphore::new(1)),
23560 &ctx,
23561 &mut state,
23562 0,
23563 )
23564 .await;
23565 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23566 assert_eq!(result[0].1.turns, 0);
23567 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23568 assert_eq!(
23569 state.resource_block.unwrap().cause,
23570 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23571 );
23572 assert!(state.active.is_empty());
23573 }
23574
23575 #[tokio::test]
23576 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23577 crate::run::pin_test_home();
23578 let tmp = tempfile::tempdir().unwrap();
23579 let cache = tmp.path().join("target-a");
23580 let guard = hold(&cache);
23581 tokio::spawn(async move {
23583 tokio::time::sleep(Duration::from_millis(1500)).await;
23584 guard.release();
23585 });
23586 let mut st = state(60);
23587 let results = run(
23588 vec![job(
23589 sh("alpha", "echo ok"),
23590 tmp.path(),
23591 Duration::from_secs(1),
23592 )],
23593 Arc::new(Semaphore::new(1)),
23594 Some(&cache),
23595 &mut st,
23596 )
23597 .await;
23598 assert!(
23599 matches!(results[0].2, AgentOutcome::Ok(_)),
23600 "the wait must not be charged to the agent"
23601 );
23602 assert!(st.resource_block.is_none());
23603 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23604 assert!(st.resource_wait.is_none() && st.active.is_empty());
23605 }
23606
23607 #[tokio::test]
23608 #[cfg(unix)]
23609 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23610 use std::os::fd::AsRawFd as _;
23611 crate::run::pin_test_home();
23612 let tmp = tempfile::tempdir().unwrap();
23613 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23614 let common = tmp.path().join(".git");
23615 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23616 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23617 assert_eq!(
23618 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23619 0
23620 );
23621 let mut st = state(1);
23622 st.config.coordination = crate::config::test_coordination();
23623 let mut reader = job(
23624 sh("reader", "echo must-not-run"),
23625 tmp.path(),
23626 Duration::from_secs(30),
23627 );
23628 reader.allow_write = false;
23629 let results = run(
23630 vec![reader.clone()],
23631 Arc::new(Semaphore::new(1)),
23632 None,
23633 &mut st,
23634 )
23635 .await;
23636 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23637 assert_eq!(results[0].1.turns, 0);
23638 assert!(FailClass::of(&results[0].2).is_none());
23639 assert_eq!(
23640 st.resource_block.as_ref().unwrap().kind,
23641 "repository coordination"
23642 );
23643 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23644 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23645 drop(external);
23646 let editor = crate::repo_lock::SeatLease::for_checkout(
23647 tmp.path(),
23648 true,
23649 crate::config::test_coordination().as_ref(),
23650 )
23651 .await
23652 .unwrap();
23653 let pause = Pause::new();
23654 let interrupt = Pause::new();
23655 let prompts = Prompts::default();
23656 let ctx = WaveCtx {
23657 run: "r",
23658 node: "review",
23659 prompts: &prompts,
23660 cache: None,
23661 round: None,
23662 carry_seats: false,
23663 pauses: Some((&pause, &interrupt)),
23664 };
23665 let (results, ()) = tokio::join!(
23666 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23667 async {
23668 tokio::time::sleep(Duration::from_millis(50)).await;
23669 pause.park();
23670 }
23671 );
23672 assert!(
23673 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23674 );
23675 assert_eq!(results[0].1.turns, 0);
23676 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23677 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23678 drop(editor);
23679 }
23680
23681 #[tokio::test]
23682 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23683 crate::run::pin_test_home();
23684 let tmp = tempfile::tempdir().unwrap();
23685 let cache = tmp.path().join("target-b");
23686 let _held = hold(&cache);
23687 let mut st = state(1);
23688 let results = run(
23689 vec![job(
23690 sh("alpha", "echo ok"),
23691 tmp.path(),
23692 Duration::from_secs(1),
23693 )],
23694 Arc::new(Semaphore::new(1)),
23695 Some(&cache),
23696 &mut st,
23697 )
23698 .await;
23699 let (_, seat, out) = &results[0];
23700 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23701 assert!(wave_blocked(&results));
23702 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23703 assert_eq!(seat.turns, 0, "nothing was asked");
23704 assert!(
23705 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23706 "no agent was spawned"
23707 );
23708 assert!(FailClass::of(out).is_none(), "not a failure class");
23709 let block = st.resource_block.as_ref().expect("recorded for the resume");
23710 assert_eq!(block.kind, "shared build cache");
23711 assert!(matches!(
23712 block.cause,
23713 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23714 ));
23715 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23716 assert!(st.active.is_empty());
23717 }
23718
23719 #[tokio::test]
23720 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23721 crate::run::pin_test_home();
23722 let tmp = tempfile::tempdir().unwrap();
23723 let mut st = state(1);
23724 let results = run(
23725 vec![job(
23726 sh("alpha", "echo ok"),
23727 tmp.path(),
23728 Duration::from_secs(1),
23729 )],
23730 Arc::new(Semaphore::new(0)),
23731 None,
23732 &mut st,
23733 )
23734 .await;
23735 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23736 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23737 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23738 }
23739
23740 #[tokio::test]
23741 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23742 crate::run::pin_test_home();
23743 let tmp = tempfile::tempdir().unwrap();
23744 let mut st = state(60);
23745 let results = run(
23746 vec![job(
23747 sh("alpha", "sleep 30"),
23748 tmp.path(),
23749 Duration::from_secs(1),
23750 )],
23751 Arc::new(Semaphore::new(1)),
23752 None,
23753 &mut st,
23754 )
23755 .await;
23756 let out = &results[0].2;
23757 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23758 let class = FailClass::of(out).expect("a failure");
23759 assert_eq!(class, FailClass::Timeout);
23760 assert!(should_hand_over(None, &class), "existing handover policy");
23761 assert!(st.resource_block.is_none());
23762 }
23763}