1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.record_base_sync_stop(&head);
1417 state.save()?;
1418 if resumption {
1419 anyhow::ensure!(
1420 crate::daemon::compatible_run_instruction(&state, &queued),
1421 "resumption instruction does not match the retained run"
1422 );
1423 let approval = queued
1424 .resumption
1425 .as_ref()
1426 .context("no operator resumption")?;
1427 anyhow::ensure!(
1428 approval.instruction == queued.instruction,
1429 "resumption approval does not match the current task instruction"
1430 );
1431 state.authorize_base_sync_resumption(task, approval)?;
1432 } else {
1433 state.authorize_base_sync(task)?;
1434 }
1435 state.event(
1436 "base-recovery",
1437 "owner authorized one additional base rebase; fresh verification required",
1438 );
1439 state.save()?;
1440 Ok(())
1441 }
1442
1443 pub fn resume(id: &str) -> Result<Self> {
1445 let state = RunState::load(id)?;
1446 if state.startup.as_ref().is_some_and(|s| !s.complete)
1447 || (state.status == RunStatus::Prep
1448 && state.instruction.trim().is_empty()
1449 && state.candidates.is_empty()
1450 && state.prep_complete == Some(false))
1451 {
1452 bail!(
1453 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1454 );
1455 }
1456 crate::pr_head::ensure_owner(&state)?;
1457 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1458 bail!(
1459 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1460 );
1461 }
1462 if let Some(to) = &state.released_to {
1463 bail!(
1464 "run {} cannot be resumed: its worktree was released to run {}",
1465 state.short(),
1466 crate::run::short_of(to)
1467 );
1468 }
1469 let roles = state.config.resolve_roles()?;
1470 if let Some(id) = &state.config.graph.selected_implementer
1471 && state.candidates.iter().any(|c| c.agent != *id)
1472 {
1473 bail!(
1474 "replacement implementer pin does not match the saved candidates; task must stay held"
1475 );
1476 }
1477 let max_parallel = state.config.graph.max_parallel.max(1);
1478 Ok(Self {
1479 state,
1480 roles,
1481 sem: Arc::new(Semaphore::new(max_parallel)),
1482 pause: Pause::new(),
1483 interrupt: Pause::new(),
1484 })
1485 }
1486
1487 pub async fn execute(&mut self) -> Result<()> {
1494 anyhow::ensure!(
1495 self.state.startup.as_ref().is_none_or(|s| s.complete),
1496 "startup incomplete; inspect retained evidence before explicit recovery"
1497 );
1498 if let Some(selection) = &self.state.pr_recovery
1499 && (selection.source_proof.is_some() || selection.source_run.is_some())
1500 {
1501 Box::pin(selection.verify_source(&self.state.repo)).await?;
1502 }
1503 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1506 || self.state.land_approval.is_some()
1507 || self.state.merge_decision.is_some()
1508 || !self.state.merge_repairs.is_empty()
1509 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1510 {
1511 let claim = FixClaim::acquire(&self.state.dir())?;
1512 let disk = RunState::load(&self.state.id)?;
1513 let checkpoint = |s: &RunState| {
1514 (
1515 s.merge_repairs.len(),
1516 s.merge_repairs
1517 .last()
1518 .map(|r| (r.question.clone(), r.phase)),
1519 s.land_approval
1520 .as_ref()
1521 .map(|a| (a.question.clone(), a.head.clone())),
1522 )
1523 };
1524 if checkpoint(&disk) != checkpoint(&self.state) {
1525 bail!(
1526 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1527 );
1528 }
1529 Some(claim)
1530 } else {
1531 None
1532 };
1533 let mut execution = self.claim_driver().await?;
1534 let runner = &mut *execution.runner;
1535 let wait = runner.repository_wait();
1536 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1539 let result = match result {
1540 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1541 record_repository_block(&mut runner.state, &e);
1542 runner.park_here()?;
1543 Ok(())
1544 }
1545 other => other,
1546 };
1547 execution.driver.finish(&mut runner.state).await?;
1548 let ended = if result.is_err() {
1549 Some({
1550 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1551 if let Err(error) = &result {
1552 notice.message = format!(
1553 "Run {} stopped with an error: {}",
1554 runner.state.id,
1555 error.to_string().lines().next().unwrap_or("unknown error")
1556 );
1557 }
1558 notice
1559 })
1560 } else {
1561 crate::notices::run_ended(&runner.state)
1562 };
1563 if let Some(notice) = ended {
1564 crate::notices::raise(notice);
1565 }
1566 result
1567 }
1568
1569 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1570 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1571 }
1572
1573 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1574 let wait = self.repository_wait();
1575 coordinate_operation(&mut self.state, cwd, &wait).await
1576 }
1577
1578 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1580 let owner = match crate::pr_head::owner_lock_wait(
1581 &crate::run::home(),
1582 &self.state.id,
1583 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1584 )
1585 .await
1586 {
1587 Ok(owner) => owner,
1588 Err(error) => {
1589 let detail = format!("worktree maintenance or ownership update: {error:#}");
1590 self.state.resource_block = Some(crate::run::ResourceBlock {
1594 cause: crate::run::ResourceBlockCause::Other,
1595 node: "start".into(),
1596 kind: "worktree maintenance".into(),
1597 detail: detail.clone(),
1598 at: Timestamp::now(),
1599 });
1600 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1601 }
1602 };
1603 let disk = RunState::load(&self.state.id)?;
1604 crate::pr_head::ensure_owner(&disk)?;
1605 let claimed =
1606 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1607 anyhow::ensure!(
1612 disk.liveness(claimed) != crate::run::Liveness::Live,
1613 "run already has a live driver; wait for its execution to end"
1614 );
1615 if let Some(to) = &disk.released_to {
1616 bail!(
1617 "run {} cannot continue: its worktree was released to run {}",
1618 self.state.short(),
1619 crate::run::short_of(to)
1620 );
1621 }
1622 anyhow::ensure!(
1623 serde_json::to_value(&disk.base_sync_recovery)?
1624 == serde_json::to_value(&self.state.base_sync_recovery)?,
1625 "base recovery changed since this run was loaded; reload the saved run"
1626 );
1627 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1628 let mut execution = DriverExecution {
1629 runner: self,
1630 driver,
1631 };
1632 let _owner = owner;
1635 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1636 .await?;
1637 let state = &mut execution.runner.state;
1638 state.parked = false;
1643 state.resource_wait = None;
1644 state.resource_block = None;
1645 state.clear_active();
1652 let pid = std::process::id();
1667 state.driver_pid = Some(pid);
1668 state.driver_started_at = crate::proc::process_started_at(pid);
1669 state.driver_exited = false;
1670 state.driver_claimed = Some(true);
1671 state.begin_pass();
1672 execution.driver.bind(state);
1673 state.save()?;
1674 execution.driver.record(state)?;
1675 Ok(execution)
1676 }
1677
1678 async fn execute_granted_repair(&mut self) -> Result<()> {
1681 use crate::repair_budget::{Kind, Phase};
1682 let index = self.state.repair_grants.len() - 1;
1683 let winner = self
1684 .state
1685 .winner()
1686 .cloned()
1687 .context("repair winner missing")?;
1688 loop {
1689 let grant = self.state.repair_grants[index].clone();
1690 match grant.phase {
1691 Phase::Pending => {
1692 if grant.spent >= grant.receipt.request.rounds {
1693 self.state.repair_grants[index].phase = Phase::Complete;
1694 self.state.status = RunStatus::Blocked;
1695 self.state.save()?;
1696 return Ok(());
1697 }
1698 self.state.repair_grants[index].spent += 1;
1701 self.state.repair_grants[index].phase = Phase::Fixing;
1702 self.state.save()?;
1703 match grant.receipt.request.kind {
1704 Kind::Gate => {
1705 let failed = self.state.gate.clone();
1706 let _ = self.gate_fix_round(&winner, &failed).await?;
1707 }
1708 Kind::Review => {
1709 let mut record = self
1710 .state
1711 .reviews
1712 .last()
1713 .cloned()
1714 .context("review failure missing")?;
1715 let round = record.round;
1716 let step = self.run_fix_round(&winner, &mut record).await?;
1717 self.state.reviews[round - 1] = record;
1718 if matches!(step, FixStep::Recorded { report_lost: true }) {
1719 self.state.repair_grants[index].phase = Phase::Complete;
1720 self.state.status = RunStatus::Blocked;
1721 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1722 self.state.save()?;
1723 return Ok(());
1724 }
1725 if matches!(step, FixStep::Resource) {
1726 self.stop_for_resource()?;
1727 return Ok(());
1728 }
1729 self.state.save()?;
1730 }
1731 }
1732 if self.resource_halt()? {
1733 return Ok(());
1734 }
1735 }
1736 Phase::Fixing => {
1737 let _coordination = self.coordinate(&winner.worktree).await?;
1740 let rescued = git::rescue_commit(
1741 &winner.worktree,
1742 "magi: preserve interrupted granted repair",
1743 )
1744 .await?;
1745 self.state.note_withheld("repair-grant", &rescued.withheld);
1746 self.state
1749 .verification_history
1750 .push(crate::run::VerificationEvidence {
1751 superseded_by: None,
1752 archived_at: Some(Timestamp::now()),
1753 fixes_spent: Some(self.state.gate_fixes.len()),
1754 gate_head: self.state.gate_head.take(),
1755 gate_ran: std::mem::take(&mut self.state.gate_ran),
1756 gate: std::mem::take(&mut self.state.gate),
1757 e2e: self.state.head_e2e.take(),
1758 });
1759 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1760 self.state
1761 .verification_history
1762 .push(crate::run::VerificationEvidence {
1763 superseded_by: None,
1764 archived_at: Some(Timestamp::now()),
1765 fixes_spent: Some(self.state.gate_fixes.len()),
1766 gate_head: None,
1767 gate_ran: false,
1768 gate: Vec::new(),
1769 e2e: Some(e2e),
1770 });
1771 }
1772 self.state.repair_grants[index].review_start = self.state.reviews.len();
1773 self.state.repair_grants[index].head =
1774 git::rev_parse(&winner.worktree, "HEAD").await?;
1775 self.state.repair_grants[index].phase = Phase::Reviewing;
1776 self.state.save()?;
1777 }
1778 Phase::Reviewing => {
1779 self.review_loop_pass().await?;
1780 if self.resource_halt()? {
1781 return Ok(());
1782 }
1783 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1784 let clean = self.state.reviews.len() > grant.review_start
1785 && self.state.reviews.last().is_some_and(|r| {
1786 r.head == head
1787 && r.clean
1788 && r.expected > 0
1789 && r.answered == r.expected
1790 && matches!(
1791 r.e2e_status(),
1792 E2eStatus::Passed | E2eStatus::NotConfigured
1793 )
1794 });
1795 if !clean {
1796 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1798 == Kind::Review
1799 && grant.spent < grant.receipt.request.rounds
1800 && self.state.reviews.last().is_some_and(|r| {
1801 r.fix.is_none()
1802 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1803 }) {
1804 Phase::Pending
1805 } else {
1806 Phase::Complete
1807 };
1808 self.state.status = RunStatus::Blocked;
1809 self.state.event(
1810 "repair-grant",
1811 "fresh review did not clear the repaired head; no publication",
1812 );
1813 self.state.save()?;
1814 if self.state.repair_grants[index].phase == Phase::Pending {
1815 continue;
1816 }
1817 return Ok(());
1818 }
1819 self.state.repair_grants[index].phase = Phase::Gating;
1820 self.state.save()?;
1821 }
1822 Phase::Gating => {
1823 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1824 self.state.reconcile_verification_head(&head);
1825 let wait = self.repository_wait();
1826 if self
1827 .state
1828 .reviews
1829 .last()
1830 .is_none_or(|r| r.head != head || !r.clean)
1831 {
1832 self.state.status = RunStatus::Blocked;
1833 self.state.repair_grants[index].phase = Phase::Complete;
1834 self.state.event(
1835 "repair-grant",
1836 "head changed since fresh review; repair held",
1837 );
1838 self.state.save()?;
1839 return Ok(());
1840 }
1841 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1842 return Ok(());
1843 }
1844 let outcomes = self.run_gate(&winner).await?;
1845 if verify_inconclusive(&outcomes) {
1846 return Ok(());
1847 }
1848 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1849 if after != head {
1850 self.state.repair_grants[index].review_start = self.state.reviews.len();
1851 self.state.repair_grants[index].head = after;
1852 self.state.repair_grants[index].phase = Phase::Reviewing;
1853 self.state.status = RunStatus::Blocked;
1854 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1855 self.state.save()?;
1856 return Ok(());
1857 }
1858 let passed = outcomes.iter().all(CommandOutcome::ok);
1859 self.state.repair_grants[index].succeeded = passed;
1860 self.state.gate = outcomes;
1861 self.state.gate_ran = true;
1862 self.state.status = if passed {
1863 RunStatus::Gating
1864 } else {
1865 RunStatus::Blocked
1866 };
1867 self.state.repair_grants[index].phase = if !passed
1868 && grant.receipt.request.kind == Kind::Gate
1869 && grant.spent < grant.receipt.request.rounds
1870 {
1871 Phase::Pending
1872 } else {
1873 Phase::Complete
1874 };
1875 if passed && self.state.config.merge.mode == MergeMode::Pr {
1876 self.state.pr_publication_pending = true;
1878 } else if passed {
1879 self.state.status = RunStatus::Ready;
1880 }
1881 self.state.save()?;
1882 if !passed {
1883 continue;
1884 }
1885 if self.state.config.merge.mode != MergeMode::Pr {
1889 self.state.status = RunStatus::Ready;
1890 self.state.event(
1891 "repair-grant",
1892 "repair verified locally; grant carries no merge authorization",
1893 );
1894 self.state.save()?;
1895 return Ok(());
1896 }
1897 return self.merge().await;
1898 }
1899 Phase::Complete => return Ok(()),
1900 }
1901 }
1902 }
1903
1904 async fn execute_graph(&mut self) -> Result<()> {
1905 self.state.configure_recovery_resume()?;
1906 if self
1907 .state
1908 .repair_grants
1909 .last()
1910 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1911 {
1912 return self.execute_granted_repair().await;
1913 }
1914 if self
1915 .state
1916 .repair_grants
1917 .last()
1918 .is_some_and(|g| !g.succeeded)
1919 {
1920 self.state.status = RunStatus::Blocked;
1921 self.state.save()?;
1922 return Ok(());
1923 }
1924 self.persist_candidates().await?;
1927 if self.state.base_sync_blocked() {
1930 self.sync_to_base().await?;
1931 if !self.state.base_sync_blocked()
1932 && self.state.status != RunStatus::AlreadyInBase
1933 && !self.resource_halt()?
1934 {
1935 self.finish_after_tally().await?;
1936 }
1937 return Ok(());
1938 }
1939 if self
1940 .state
1941 .merge_decision
1942 .as_ref()
1943 .is_some_and(|decision| !decision.closed)
1944 && self.state.status != RunStatus::Landing
1945 {
1946 if crate::merge_decision::resumable_hold(&self.state) {
1947 self.state.merge = Some(MergeOutcome {
1948 mode: MergeMode::Pr,
1949 ok: false,
1950 detail: self
1951 .state
1952 .merge_decision
1953 .as_ref()
1954 .unwrap()
1955 .target
1956 .pr
1957 .clone(),
1958 empty: false,
1959 });
1960 self.run_land().await?;
1961 self.settle_questions();
1962 return Ok(());
1963 }
1964 bail!(
1965 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1966 );
1967 }
1968 if self.state.config.graph.selected_implementer.is_some()
1969 && self.state.status == RunStatus::Blocked
1970 && self.state.tally.is_none()
1971 && self.state.candidate_preservation.is_empty()
1972 && self.state.interrupted_implementations.is_empty()
1973 {
1974 bail!(
1975 "selected implementation is held; explicitly release the owning task to retry it"
1976 );
1977 }
1978
1979 if self.state.merge_repairs.last().is_some_and(|r| {
1980 matches!(
1981 r.phase,
1982 crate::run::MergeRepairPhase::Requested
1983 | crate::run::MergeRepairPhase::Fixing
1984 | crate::run::MergeRepairPhase::Reviewing
1985 )
1986 }) {
1987 if self.repair_merge_review().await? {
1988 self.run_land().await?;
1989 }
1990 self.settle_questions();
1991 return Ok(());
1992 }
1993 if self.state.pr_publication_pending {
1996 self.merge().await?;
1997 if self.resource_halt()? {
1998 return Ok(());
1999 }
2000 self.state.save()?;
2001 self.settle_questions();
2002 return Ok(());
2003 }
2004 if self.state.status == RunStatus::Stalled {
2017 if self.recover_stall().await? {
2018 self.finish_after_tally().await?;
2019 } else {
2020 self.state.save()?;
2022 }
2023 return Ok(());
2024 }
2025 if self.state.status == RunStatus::Landing
2035 || crate::handover::landing_checkpoint(&self.state)
2036 {
2037 self.state.land_sync_pending = true;
2038 self.state.save()?;
2039 self.run_land().await?;
2040 self.settle_questions();
2045 return Ok(());
2046 }
2047 if self.state.github_text.is_some()
2052 && self.state.merge.is_none()
2053 && self.state.status == RunStatus::Gating
2054 {
2055 self.merge().await?;
2056 if self.resource_halt()? {
2057 return Ok(());
2058 }
2059 self.state.save()?;
2060 return Ok(());
2061 }
2062 if let Err(e) = self.prep().await {
2063 if e.is::<crate::repo_lock::ResourceBlocked>() {
2064 return Err(e);
2065 }
2066 self.state.status = RunStatus::Blocked;
2067 self.state
2068 .prep_failure
2069 .get_or_insert_with(|| format!("{e:#}"));
2070 self.state
2071 .event("prep", format!("preparation stopped: {e:#}"));
2072 self.state.save()?;
2073 return Err(e);
2074 }
2075 if self.park_here()? {
2076 return Ok(());
2077 }
2078 self.advise().await?;
2079 if self.park_here()? {
2080 return Ok(());
2081 }
2082 self.implement().await?;
2083 if self.park_here()? {
2084 return Ok(());
2085 }
2086 if self.state.status == RunStatus::VerifiedNoop {
2090 return Ok(());
2091 }
2092 self.judge().await?;
2093 if self.park_here()? {
2094 return Ok(());
2095 }
2096 self.deliberate().await?;
2097 if self.park_here()? {
2098 return Ok(());
2099 }
2100 self.vote().await?;
2101 if self.park_here()? {
2102 return Ok(());
2103 }
2104 self.tally()?;
2105 if self.state.status == RunStatus::Stalled {
2110 self.state.save()?;
2114 return Ok(());
2115 }
2116 self.finish_after_tally().await?;
2117 Ok(())
2118 }
2119
2120 fn stop_for_resource(&mut self) -> Result<()> {
2125 if self.pause.parked() || self.interrupt.parked() {
2128 self.park_here()?;
2129 return Ok(());
2130 }
2131 if self.state.resource_block.is_some() {
2132 self.state.status = RunStatus::Blocked;
2133 }
2134 self.state.save()
2135 }
2136
2137 fn resource_halt(&mut self) -> Result<bool> {
2139 if self.state.resource_block.is_none() {
2140 return Ok(false);
2141 }
2142 self.stop_for_resource()?;
2143 Ok(true)
2144 }
2145
2146 fn park_here(&mut self) -> Result<bool> {
2153 let parking = self.pause.parked() || self.interrupt.parked();
2154 if !parking && self.state.resource_block.is_some() {
2155 self.stop_for_resource()?;
2156 return Ok(true);
2157 }
2158 if !self.pause.parked() && !self.interrupt.parked() {
2163 return Ok(false);
2164 }
2165 if self.state.parked {
2166 return Ok(true);
2167 }
2168 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2169 if let Some(block) = &mut self.state.resource_block
2170 && matches!(
2171 block.cause,
2172 crate::run::ResourceBlockCause::Cancelled { .. }
2173 )
2174 {
2175 block.cause = crate::run::ResourceBlockCause::Cancelled {
2176 park: cause.clone(),
2177 };
2178 }
2179 let why = match cause.as_ref().map(ParkCause::label) {
2180 Some(reason) => format!(
2181 "parked after `{}` ({reason}) — resume to carry on from here",
2182 self.state.status.as_str()
2183 ),
2184 None => format!(
2185 "parked after `{}` — resume to carry on from here",
2186 self.state.status.as_str()
2187 ),
2188 };
2189 self.state.event("park", why);
2190 self.state
2191 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2192 self.state.parked = true;
2193 self.state.save()?;
2194 Ok(true)
2195 }
2196
2197 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2199 if self.state.prep_failure.is_none() {
2200 return Ok(());
2201 }
2202 let failed_at = self
2203 .state
2204 .events
2205 .iter()
2206 .rev()
2207 .find(|event| event.node == "prep")
2208 .map_or(self.state.updated_at, |event| event.at);
2209 anyhow::ensure!(
2210 approval_at > failed_at,
2211 "preparation retry requires a new explicit release after the failure"
2212 );
2213 self.state.prep_failure = None;
2214 self.state.prep_complete = Some(false);
2215 self.state
2216 .event("prep-retry", "operator approved retry of preparation");
2217 Ok(())
2218 }
2219
2220 pub fn on_pause(&mut self, pause: Pause) {
2222 self.pause = pause;
2223 }
2224
2225 pub fn watch_interrupt(&mut self, pause: Pause) {
2231 self.interrupt = pause;
2232 }
2233
2234 fn settle_questions(&mut self) {
2254 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2255 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2256 }
2257 }
2258
2259 async fn finish_after_tally(&mut self) -> Result<()> {
2262 self.graft().await?;
2265 if self.resource_halt()? {
2266 return Ok(());
2267 }
2268 self.fold_losers().await?;
2269 self.sync_to_base().await?;
2274 if self.hold_base_sync().await? || self.resource_halt()? {
2275 return Ok(());
2276 }
2277 if self.state.status == RunStatus::AlreadyInBase {
2278 return Ok(());
2279 }
2280 self.review_loop().await?;
2281 if self.resource_halt()? {
2282 return Ok(());
2283 }
2284 self.sync_to_base().await?;
2285 if self.hold_base_sync().await? || self.resource_halt()? {
2286 return Ok(());
2287 }
2288 if self.state.status == RunStatus::AlreadyInBase {
2289 return Ok(());
2290 }
2291 self.gate().await?;
2292 self.merge().await?;
2293 if self.resource_halt()? {
2294 return Ok(());
2295 }
2296 self.state.save()?;
2297 Ok(())
2298 }
2299
2300 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2307 let result = match ensure_setup_config(&mut self.state, repo).await {
2308 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2309 Err(e) => Err(e),
2310 };
2311 if let Err(e) = result {
2312 git::worktree_remove_clean(repo, wt).await.ok();
2313 if in_prep {
2314 self.rollback_prep(repo).await;
2315 }
2316 self.state.status = RunStatus::Blocked;
2317 self.state.event(
2318 "setup",
2319 format!("worktree setup failed in {}: {e:#}", wt.display()),
2320 );
2321 self.state.save()?;
2322 return Err(e);
2323 }
2324 Ok(())
2325 }
2326
2327 async fn rollback_prep(&mut self, repo: &Path) {
2329 let root = self.state.worktree_root();
2330 for c in std::mem::take(&mut self.state.candidates) {
2331 if matches!(
2332 git::worktree_remove_clean(repo, &c.worktree).await,
2333 Ok(true)
2334 ) {
2335 git::branch_delete(repo, &c.branch).await.ok();
2336 } else {
2337 self.state.candidates.push(c);
2338 }
2339 }
2340 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2341 for j in 1..=seats {
2342 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2343 .await
2344 .ok();
2345 }
2346 for k in 1..=self.state.config.graph.advisors {
2347 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2348 .await
2349 .ok();
2350 }
2351 }
2352
2353 async fn prep(&mut self) -> Result<()> {
2354 if let Some(reason) = &self.state.prep_failure {
2355 bail!(
2356 "candidate preparation previously failed: {reason}; correct the source before retrying"
2357 );
2358 }
2359 if self
2360 .state
2361 .prep_complete
2362 .unwrap_or(!self.state.candidates.is_empty())
2363 {
2364 return Ok(());
2365 }
2366 let repo = self.state.repo.clone();
2367 let _coordination = self.coordinate(&repo).await?;
2370 let base = self.state.base_commit.clone();
2371 let plan = self.state.sources.plan(&repo, &base).await?;
2372 self.rollback_prep(&repo).await;
2373 if !self.state.candidates.is_empty() {
2374 bail!(
2375 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2376 );
2377 }
2378 self.state.prep_complete = Some(false);
2379 self.state.status = RunStatus::Prep;
2380 self.state.save()?;
2381 let repo = self.state.repo.clone();
2382 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2383 let root = self.state.worktree_root();
2384 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2385
2386 let hooks_dir = self.state.dir().join("hooks");
2389 if self.state.config.blind.commit_msg_hook {
2390 std::fs::create_dir_all(&hooks_dir)
2391 .with_context(|| format!("create {}", hooks_dir.display()))?;
2392 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2393 let path = hooks_dir.join("commit-msg");
2394 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2395 make_executable(&path)?;
2396 git::acquire_worktree_config(&repo).await?;
2404 self.state.enabled_worktree_config = true;
2405 }
2406
2407 ensure_setup_config(&mut self.state, &repo).await?;
2410
2411 for (index, (spec, label)) in self
2412 .roles
2413 .implementers
2414 .clone()
2415 .into_iter()
2416 .zip(labels)
2417 .enumerate()
2418 {
2419 let branch = self.state.branch_for(label);
2420 let worktree = root.join(format!("cand-{label}"));
2421 self.state.event(
2425 "prep-intent",
2426 serde_json::json!({
2427 "branch": branch,
2428 "worktree": worktree,
2429 "start": start,
2430 "picks": plan.picks,
2431 })
2432 .to_string(),
2433 );
2434 self.state.save()?;
2435 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2436 if self.state.config.blind.commit_msg_hook {
2437 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2438 }
2439 git::local_exclude(&worktree, "/.magi/").await?;
2440 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2441 git::worktree_remove_clean(&repo, &worktree).await.ok();
2444 git::branch_delete(&repo, &branch).await.ok();
2445 self.rollback_prep(&repo).await;
2446 self.state.status = RunStatus::Blocked;
2447 self.state
2448 .event("prep", format!("worktree setup failed: {e:#}"));
2449 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2450 self.state.save()?;
2451 return Err(e);
2452 }
2453 self.state.candidates.push(Candidate {
2454 index,
2455 label,
2456 agent: spec.id.clone(),
2457 branch,
2458 worktree,
2459 summary: String::new(),
2460 stat: String::new(),
2461 files: 0,
2462 commits: 0,
2463 empty: false,
2464 failed: None,
2465 verified_noop: None,
2466 duration_ms: 0,
2467 folded: false,
2468 });
2469 self.state.save()?;
2470 for pick in &plan.picks {
2471 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2472 {
2473 self.state.status = RunStatus::Blocked;
2474 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2475 self.state
2476 .event("prep", format!("cannot apply referenced commit: {e}"));
2477 self.state.save()?;
2478 return Err(e);
2479 }
2480 }
2481 }
2482
2483 for j in 1..=self.roles.judges.len() {
2484 let wt = root.join(format!("judge-{j}"));
2485 if !wt.exists() {
2486 git::worktree_add_detached(&repo, &wt, &base).await?;
2487 self.setup_seat_worktree(&repo, &wt, true).await?;
2488 }
2489 }
2490
2491 if self.state.config.graph.advise {
2499 for k in 1..=self.state.config.graph.advisors {
2500 let wt = root.join(format!("advisor-{k}"));
2501 if !wt.exists() {
2502 git::worktree_add_detached(&repo, &wt, &base).await?;
2503 self.setup_seat_worktree(&repo, &wt, true).await?;
2504 }
2505 }
2506 }
2507
2508 let authors: Vec<&str> = self
2513 .roles
2514 .implementers
2515 .iter()
2516 .map(|a| a.id.as_str())
2517 .collect();
2518 let overlap: Vec<String> = self
2519 .roles
2520 .judges
2521 .iter()
2522 .enumerate()
2523 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2524 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2525 .collect();
2526 if !overlap.is_empty() {
2527 let note = format!(
2528 "{} also authored a candidate; blind, but the panel is less \
2529 independent than {} distinct agents would be",
2530 overlap.join(", "),
2531 self.roles.judges.len()
2532 );
2533 self.state.event("prep", note);
2534 }
2535
2536 self.state.event(
2537 "prep",
2538 format!(
2539 "{} candidates, {} judges, base {} ({})",
2540 self.state.candidates.len(),
2541 self.roles.judges.len(),
2542 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2543 self.state.base_branch
2544 ),
2545 );
2546 self.state.prep_complete = Some(true);
2547 self.state.status = RunStatus::Implementing;
2548 self.state.save()?;
2549 Ok(())
2550 }
2551
2552 async fn advise(&mut self) -> Result<()> {
2585 let implement_untouched = self
2586 .state
2587 .candidates
2588 .iter()
2589 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2590 if !self.state.config.graph.advise || self.state.advise_attempted {
2591 return Ok(());
2592 }
2593 if !implement_untouched {
2594 self.state.event(
2595 "advise",
2596 "skipping the design-deliberation stage: at least one \
2597 candidate already shows implementation progress, so this \
2598 run is past the point the stage exists to run before"
2599 .to_owned(),
2600 );
2601 self.state.advise_attempted = true;
2602 self.state.save()?;
2603 return Ok(());
2604 }
2605 let run_id = self.state.id.clone();
2606 let prompts = self.state.config.prompts.clone();
2607 let instruction = self.state.instruction.clone();
2608 let language = self.state.config.graph.language.clone();
2609 let root = self.state.worktree_root();
2610 let n = self.state.config.graph.advisors;
2611 let where_recorded = self.state.dir().join("run.json");
2612
2613 let seats = match self.state.config.advisors() {
2614 Ok(seats) if !seats.is_empty() => seats,
2615 Ok(_) => {
2616 self.state.event(
2617 "advise",
2618 format!(
2619 "[graph] advisors is 0; skipping the design-deliberation \
2620 stage and continuing without a synthesis brief (see {})",
2621 where_recorded.display()
2622 ),
2623 );
2624 self.state.advise_attempted = true;
2625 self.state.save()?;
2626 return Ok(());
2627 }
2628 Err(e) => {
2629 self.state.event(
2630 "advise",
2631 format!(
2632 "could not resolve advisor seats ({e:#}); continuing \
2633 without a design-deliberation brief (see {})",
2634 where_recorded.display()
2635 ),
2636 );
2637 self.state.advise_attempted = true;
2638 self.state.save()?;
2639 return Ok(());
2640 }
2641 };
2642
2643 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2644 let artifacts = agent::artifacts_dir(&self.state.dir());
2645 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2646
2647 let mut jobs = Vec::new();
2648 for (i, spec) in seats.iter().cloned().enumerate() {
2649 let seat_key = format!("advisor-{}", i + 1);
2650 let seat = self.seat(&seat_key, &spec.id);
2651 jobs.push(SeatJob {
2652 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2653 spec,
2654 seat,
2655 cwd: worktrees[i % worktrees.len()].clone(),
2656 timeout,
2657 allow_write: false,
2658 sessions: false,
2659 artifacts: artifacts.clone(),
2660 stem: seat_key,
2661 handover: None,
2662 });
2663 }
2664
2665 self.state.event(
2666 "advise",
2667 format!(
2668 "{} advisor seat(s) sketching a design in parallel",
2669 jobs.len()
2670 ),
2671 );
2672 let mut quota_losses = Vec::new();
2673 let cache = self.state.config.cache_dir();
2674 let ctx = WaveCtx {
2675 carry_seats: false,
2676 pauses: Some((&self.pause, &self.interrupt)),
2677 run: &run_id,
2678 node: "advise",
2679 prompts: &prompts,
2680 cache: cache.as_deref(),
2681 round: None,
2682 };
2683 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2684 let results = ask_json_wave::<Proposal>(
2685 jobs,
2686 Arc::clone(&self.sem),
2687 self.state.config.graph.retries,
2688 &advisor_roster,
2689 &ctx,
2690 &mut quota_losses,
2691 &mut self.state,
2692 &|p: &Proposal| p.validate(),
2693 )
2694 .await;
2695 if self.state.resource_block.is_some() {
2696 return self.stop_for_resource();
2697 }
2698 self.state.quota.extend(quota_losses);
2699
2700 let mut records = Vec::with_capacity(results.len());
2701 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2702 let agent_id = seat.agent.clone();
2703 self.state.seats.insert(seat.key.clone(), seat);
2704 match res {
2705 Ok((proposal, out)) => {
2706 self.state
2707 .event("advise", format!("advisor-{} proposed a design", i + 1));
2708 records.push(advise::AdvisorRecord::proposed(
2709 i + 1,
2710 agent_id,
2711 proposal,
2712 out.duration_ms,
2713 ));
2714 }
2715 Err(e) => {
2716 self.state.event(
2717 "advise",
2718 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2719 );
2720 records.push(advise::AdvisorRecord::failed(
2721 i + 1,
2722 agent_id,
2723 e.to_string(),
2724 ));
2725 }
2726 }
2727 }
2728
2729 let mut advice = advise::Advice {
2730 records,
2731 synthesis: None,
2732 };
2733 if advice.proposals().is_empty() {
2734 self.state.event(
2735 "advise",
2736 "no advisor produced a usable proposal; continuing without a \
2737 synthesis brief"
2738 .to_owned(),
2739 );
2740 } else {
2741 match self
2742 .synthesize_brief(
2743 &advice,
2744 &instruction,
2745 &language,
2746 &worktrees[0],
2747 &artifacts,
2748 &run_id,
2749 &prompts,
2750 cache.as_deref(),
2751 )
2752 .await
2753 {
2754 Ok(Some(text)) => {
2755 self.state.event(
2756 "advise",
2757 "synthesized a design brief for the implementer".to_owned(),
2758 );
2759 advice.synthesis = Some(text);
2760 }
2761 Ok(None) => {
2762 self.state.event(
2763 "advise",
2764 "the synthesis seat produced nothing usable; continuing \
2765 without a design brief"
2766 .to_owned(),
2767 );
2768 }
2769 Err(e) => {
2770 if e.is::<crate::repo_lock::ResourceBlocked>() {
2771 return Err(e);
2772 }
2773 self.state.event(
2774 "advise",
2775 format!("could not synthesize a design brief: {e:#}"),
2776 );
2777 }
2778 }
2779 }
2780 advise::apply_reflection(&mut advice);
2781
2782 self.state.advice = Some(advice);
2783 self.state.advise_attempted = true;
2784 self.state.save()?;
2785 Ok(())
2786 }
2787
2788 #[allow(clippy::too_many_arguments)]
2800 async fn synthesize_brief(
2801 &mut self,
2802 advice: &advise::Advice,
2803 instruction: &str,
2804 language: &str,
2805 cwd: &Path,
2806 artifacts: &Path,
2807 run_id: &str,
2808 prompts: &Prompts,
2809 cache: Option<&Path>,
2810 ) -> Result<Option<String>> {
2811 let chain = agent::pick_chain(
2812 &self.state.config.agents,
2813 self.state.config.roles.synthesizer.as_ref(),
2814 &agent::installed,
2815 "synthesizer",
2816 )?;
2817 let proposals = advice.proposals();
2818 let mut prompt = prompt::with_overlay(
2819 prompt::synthesize_brief(instruction, &proposals, language),
2820 prompts.overlay("advise"),
2821 );
2822 if cache.is_some() {
2823 prompt.push('\n');
2828 prompt.push_str(&prompt::build_cache_note("advise", false));
2829 }
2830 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2831 let mut last = None;
2836 for (n, spec) in chain.iter().enumerate() {
2837 if n > 0 {
2838 self.state
2839 .event("advise", format!("synthesis falling back to {}", spec.id));
2840 }
2841 let mut seat = self.seat("advise-synthesis", &spec.id);
2842 let wait = self.repository_wait();
2843 let outcome = wait
2844 .scope(agent::invoke(
2845 spec,
2846 &mut seat,
2847 &Invocation {
2848 coordination: self.state.config.coordination.as_ref(),
2849 fetched_base: None,
2850 cwd,
2851 prompt: &prompt,
2852 timeout,
2853 allow_write: false,
2854 sessions: false,
2855 artifacts,
2856 stem: &if n == 0 {
2857 "advise-synthesis".to_owned()
2858 } else {
2859 format!("advise-synthesis-{}", spec.id)
2860 },
2861 run: run_id,
2862 node: "advise",
2863 cache_dir: None,
2864 attachments: &[],
2865 writable: &[],
2866 },
2867 ))
2868 .await;
2869 if outcome.is_ok() {
2870 self.state.seats.insert(seat.key.clone(), seat);
2871 }
2872 let advance = agent::chain_advances(&outcome);
2873 last = Some(outcome);
2874 if !advance {
2875 break;
2876 }
2877 }
2878 let out = last.expect("a chain holds at least one agent")?;
2881 if !out.usable() {
2882 return Ok(None);
2883 }
2884 let text =
2885 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2886 Ok((!text.trim().is_empty()).then_some(text))
2887 }
2888
2889 async fn implement(&mut self) -> Result<()> {
2892 self.persist_candidates().await?;
2893 let run_id = self.state.id.clone();
2898 let prompts = self.state.config.prompts.clone();
2899 let todo: Vec<usize> = self
2900 .state
2901 .candidates
2902 .iter()
2903 .enumerate()
2904 .filter(|(_, c)| {
2905 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2906 && c.failed.is_none()
2907 && !c.empty
2908 })
2909 .map(|(i, _)| i)
2910 .collect();
2911 if todo.is_empty() {
2912 return self.after_implement();
2913 }
2914 self.state.status = RunStatus::Implementing;
2915
2916 let language = self.state.config.graph.language.clone();
2917 let github_language = self.state.config.graph.github_language.clone();
2918 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2919 let sessions = self.state.config.graph.sessions;
2920 let artifacts = agent::artifacts_dir(&self.state.dir());
2921 let brief = self
2925 .state
2926 .advice
2927 .as_ref()
2928 .and_then(|a| a.synthesis.as_deref())
2929 .map(str::to_owned);
2930 let attachments = self.state.attachments.clone();
2931
2932 let mut jobs = Vec::new();
2933 for &i in &todo {
2934 let (index, label, worktree) = {
2935 let c = &self.state.candidates[i];
2936 (c.index, c.label, c.worktree.clone())
2937 };
2938 let spec = self.roles.implementers[index].clone();
2939 let seat_key = format!("impl-{label}");
2940 let seat = self.seat(&seat_key, &spec.id);
2941 let instruction = seeded_instruction(&self.state);
2942 jobs.push(SeatJob {
2943 spec,
2944 seat,
2945 prompt: prompt::implement_in(
2946 &instruction,
2947 &worktree.to_string_lossy(),
2948 &language,
2949 &github_language,
2950 brief.as_deref(),
2951 &attachments,
2952 ),
2953 cwd: worktree,
2954 timeout,
2955 allow_write: true,
2956 sessions,
2957 artifacts: artifacts.clone(),
2958 stem: format!("impl-{label}"),
2959 handover: None,
2960 });
2961 }
2962
2963 self.state.event(
2964 "implement",
2965 format!("{} candidates in parallel", jobs.len()),
2966 );
2967 let mut sent = jobs.clone();
2973 let cache = self.state.config.cache_dir();
2974 let ctx = WaveCtx {
2975 carry_seats: false,
2976 pauses: Some((&self.pause, &self.interrupt)),
2977 run: &run_id,
2978 node: "implement",
2979 prompts: &prompts,
2980 cache: cache.as_deref(),
2981 round: None,
2982 };
2983 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2984 if !wave_blocked(&results) {
2985 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2986 .await;
2987 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2988 .await?;
2989 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2990 .await;
2991 }
2992 if !results.is_empty()
2995 && results
2996 .iter()
2997 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
2998 {
2999 let losses: Vec<_> = results
3000 .iter()
3001 .map(|(_, seat, out)| {
3002 let AgentOutcome::Quota(o) = out else {
3003 unreachable!()
3004 };
3005 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3006 })
3007 .collect();
3008 self.state.quota.extend(losses.clone());
3009 for (_, seat, _) in results {
3010 self.state.seats.insert(seat.key.clone(), seat);
3011 }
3012 for &i in &todo {
3013 let label = self.state.candidates[i].label;
3014 self.state.interrupted_implementations.insert(label);
3015 self.state
3016 .candidate_preservation
3017 .insert(label, "machine outage awaiting persistence".into());
3018 }
3019 block_machine_outage(&mut self.state, "implement", &losses);
3022 self.state.save()?;
3023 self.persist_candidates().await?;
3024 for &i in &todo {
3025 self.state
3026 .interrupted_implementations
3027 .insert(self.state.candidates[i].label);
3028 }
3029 return self.stop_for_resource();
3030 }
3031 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3032
3033 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3034 && results
3035 .iter()
3036 .any(|(_, _, out)| FailClass::of(out).is_some());
3037 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3038 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3039 if self
3043 .state
3044 .candidate_preservation
3045 .contains_key(&self.state.candidates[i].label)
3046 {
3047 self.state.seats.insert(seat.key.clone(), seat);
3048 }
3049 continue;
3050 }
3051 let seat_key = seat.key.clone();
3052 let agent = seat.agent.clone();
3061 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3062 self.state.seats.insert(seat.key.clone(), seat);
3063 let label = self.state.candidates[i].label;
3064 self.state.interrupted_implementations.remove(&label);
3065
3066 let (summary, duration, failed, verified_claim) = match out {
3067 AgentOutcome::Ok(o) => {
3068 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3069 let failed = (!o.usable()).then(|| {
3070 if o.timed_out {
3071 "agent timed out".to_owned()
3072 } else {
3073 format!(
3074 "agent exited with {}",
3075 crate::notices::exit_status(o.exit_code)
3076 )
3077 }
3078 });
3079 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3080 (text, o.duration_ms, failed, verified_claim)
3081 }
3082 AgentOutcome::Dropped(o) => {
3088 let why = o
3089 .dropped
3090 .as_ref()
3091 .map(|d| d.why.as_str())
3092 .unwrap_or("the CLI ended the stream without delivering its answer");
3093 (
3094 String::new(),
3095 o.duration_ms,
3096 Some(format!("the CLI dropped the stream ({why})")),
3097 None,
3098 )
3099 }
3100 AgentOutcome::Quota(o) => {
3101 self.state.quota.push(QuotaLoss::from_report(
3102 seat_key,
3103 "implement",
3104 o.quota.as_ref(),
3105 ));
3106 (
3107 String::new(),
3108 o.duration_ms,
3109 Some("rate limited (quota); produced no change".to_owned()),
3110 None,
3111 )
3112 }
3113 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3114 (String::new(), 0, Some(e), None)
3115 }
3116 };
3117
3118 let c = &mut self.state.candidates[i];
3121 if !exhausted_the_fallback_chain {
3122 c.agent = agent;
3123 }
3124 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3125 c.duration_ms = duration;
3126 c.failed = failed;
3127 c.verified_noop = verified_claim;
3128 self.state
3129 .candidate_preservation
3130 .insert(label, "awaiting persistence".to_owned());
3131 }
3132 self.state.save()?;
3133 self.persist_candidates().await?;
3134
3135 if resource_blocked {
3136 return self.stop_for_resource();
3137 }
3138 if selected_failed {
3139 self.state.status = RunStatus::Blocked;
3140 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3141 self.state.save()?;
3142 bail!(
3143 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3144 );
3145 }
3146 self.after_implement()
3147 }
3148
3149 async fn persist_candidates(&mut self) -> Result<()> {
3151 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3152 for label in labels {
3153 let i = self
3154 .state
3155 .candidates
3156 .iter()
3157 .position(|c| c.label == label)
3158 .context("pending candidate is missing")?;
3159 let c = self.state.candidates[i].clone();
3160 let interrupted = self.state.interrupted_implementations.contains(&label);
3161 let result = async {
3162 let wait = crate::repo_lock::WaitPolicy::new(
3163 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3164 || false,
3165 );
3166 let _coordination =
3167 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3168 if let Err(e) = git::snapshot_uncommitted(
3171 &c.worktree,
3172 &self
3173 .state
3174 .dir()
3175 .join("artifacts")
3176 .join(format!("candidate-{label}-recovery")),
3177 )
3178 .await
3179 {
3180 self.state.event(
3181 "preserve",
3182 format!("candidate {label} snapshot incomplete: {e:#}"),
3183 );
3184 }
3185 let rescued = git::rescue_commit(
3186 &c.worktree,
3187 &format!("magi: candidate {label} (uncommitted work)"),
3188 )
3189 .await?;
3190 self.state.note_withheld("implement", &rescued.withheld);
3191 let base = &self.state.base_commit;
3192 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3193 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3194 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3195 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3196 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3197 if c.empty && commits > 0 && !patch.trim().is_empty()
3198 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3199 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3200 || self.state.tally.is_some())
3201 {
3202 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3206 }
3207 let c = &mut self.state.candidates[i];
3208 c.commits = commits;
3209 c.stat = stat;
3210 c.files = files;
3211 let no_changes = commits == 0 || patch.trim().is_empty();
3212 c.empty = no_changes && !interrupted;
3213 if interrupted && no_changes {
3214 c.failed = None;
3215 c.verified_noop = None;
3216 }
3217 if !c.empty {
3218 c.failed = None;
3219 c.verified_noop = None;
3220 }
3221 self.state.event(
3222 "preserve",
3223 format!("candidate {label}: {files} files, {commits} commits"),
3224 );
3225 Ok::<_, anyhow::Error>(())
3226 }
3227 .await;
3228 match result {
3229 Ok(()) => {
3230 self.state.candidate_preservation.remove(&label);
3231 if !interrupted || self.state.candidates[i].files > 0 {
3232 self.state.interrupted_implementations.remove(&label);
3233 }
3234 }
3235 Err(e) => {
3236 let reason = format!("candidate {label} preservation failed: {e:#}");
3237 self.state.event("preserve", &reason);
3238 self.state.candidate_preservation.insert(label, reason);
3239 }
3240 }
3241 if self.state.candidate_preservation.is_empty()
3242 && self.state.status == RunStatus::Blocked
3243 && self.state.candidates.iter().all(|c| c.failed.is_none())
3244 {
3245 self.state.status = RunStatus::Implementing;
3249 self.state.event(
3250 "preserve",
3251 "candidate persistence recovered; implementation can continue",
3252 );
3253 }
3254 self.state.save()?;
3255 }
3256 if !self.state.candidate_preservation.is_empty() {
3257 self.state.status = RunStatus::Blocked;
3258 self.state.save()?;
3259 bail!(
3260 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3261 );
3262 }
3263 Ok(())
3264 }
3265
3266 async fn resume_undelivered(
3294 &mut self,
3295 results: &mut [(usize, SeatState, AgentOutcome)],
3296 sent: &[SeatJob],
3297 prompts: &Prompts,
3298 run_id: &str,
3299 ) {
3300 for (wi, seat, out) in results.iter_mut() {
3301 let Some(dropped) = (match &*out {
3302 AgentOutcome::Dropped(o) => o.dropped.clone(),
3303 _ => None,
3304 }) else {
3305 continue;
3306 };
3307 let Some(job) = sent.get(*wi) else { continue };
3308 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3310 self.state.event(
3311 "implement",
3312 format!(
3313 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3314 work is in the tree",
3315 seat.key, dropped.output_tokens, dropped.why
3316 ),
3317 );
3318 continue;
3319 }
3320 if !has_context(&job.spec, seat, job.sessions) {
3328 self.state.event(
3329 "implement",
3330 format!(
3331 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3332 is no session left to resume",
3333 seat.key, dropped.output_tokens, dropped.why
3334 ),
3335 );
3336 continue;
3337 }
3338 self.state.event(
3339 "implement",
3340 format!(
3341 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3342 conversation",
3343 seat.key, dropped.output_tokens, dropped.why
3344 ),
3345 );
3346 let mut retry = job.clone();
3347 retry.seat = seat.clone();
3348 retry.prompt = prompt::resume_after_drop(&dropped.why);
3349 retry.timeout = retry_budget(job.timeout, true);
3350 retry.stem = format!("{}-resume", job.stem);
3351 let cache = self.state.config.cache_dir();
3352 let ctx = WaveCtx {
3353 carry_seats: false,
3354 pauses: Some((&self.pause, &self.interrupt)),
3355 run: run_id,
3356 node: "implement",
3357 prompts,
3358 cache: cache.as_deref(),
3359 round: None,
3360 };
3361 let (resumed_seat, resumed) =
3362 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3363 *seat = resumed_seat;
3364 *out = resumed;
3365 }
3366 }
3367
3368 async fn resume_seat_handovers(
3433 &mut self,
3434 results: &mut [(usize, SeatState, AgentOutcome)],
3435 sent: &mut [SeatJob],
3436 prompts: &Prompts,
3437 run_id: &str,
3438 ) -> Result<()> {
3439 let instruction = seeded_instruction(&self.state);
3440 let language = self.state.config.graph.language.clone();
3441 let github_language = self.state.config.graph.github_language.clone();
3442 let brief = self
3443 .state
3444 .advice
3445 .as_ref()
3446 .and_then(|a| a.synthesis.as_deref())
3447 .map(str::to_owned);
3448 let attachments = self.state.attachments.clone();
3449 for (wi, seat, out) in results.iter_mut() {
3450 let others: BTreeSet<String> = sent
3453 .iter()
3454 .enumerate()
3455 .filter(|(j, _)| j != wi)
3456 .map(|(_, j)| j.spec.id.clone())
3457 .collect();
3458 let Some(job) = sent.get_mut(*wi) else {
3459 continue;
3460 };
3461 let start = self
3466 .roles
3467 .implementer_roster
3468 .iter()
3469 .position(|s| s.id == job.spec.id)
3470 .unwrap_or(0);
3471 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3472 let mut fallback_attempt = 0usize;
3473 let mut prev: Option<FailClass> = None;
3474 while let Some(cur) = FailClass::of(&*out) {
3475 if !should_hand_over(prev.as_ref(), &cur) {
3476 break;
3477 }
3478 let Some(next) =
3479 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3480 .cloned()
3481 else {
3482 break;
3483 };
3484 tried.insert(next.id.clone());
3485 fallback_attempt += 1;
3486
3487 let coordination = self.coordinate(&job.cwd).await?;
3488 if let Ok(r) = git::rescue_commit(
3489 &job.cwd,
3490 &format!(
3491 "magi: candidate {} (uncommitted work before {} fallback)",
3492 seat.key,
3493 if cur == FailClass::Quota {
3494 "quota"
3495 } else {
3496 "handover"
3497 }
3498 ),
3499 )
3500 .await
3501 {
3502 self.state.note_withheld("implement", &r.withheld);
3503 }
3504 drop(coordination);
3505
3506 record_handover(
3507 &mut self.state,
3508 "implement",
3509 &seat.key,
3510 &seat.agent,
3511 &next.id,
3512 &cur,
3513 &fail_reason(&*out),
3514 );
3515 prev = Some(cur.clone());
3516
3517 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3518 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3519 job.spec = next.clone();
3527 let mut retry = job.clone();
3528 retry.seat = new_seat;
3529 retry.prompt = prompt::implement_in(
3530 &instruction,
3531 &job.cwd.to_string_lossy(),
3532 &language,
3533 &github_language,
3534 brief.as_deref(),
3535 &attachments,
3536 );
3537 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3538 let cache = self.state.config.cache_dir();
3539 let ctx = WaveCtx {
3540 carry_seats: false,
3541 pauses: Some((&self.pause, &self.interrupt)),
3542 run: run_id,
3543 node: "implement",
3544 prompts,
3545 cache: cache.as_deref(),
3546 round: None,
3547 };
3548 let (fallback_seat, fallback_out) = run_one(
3549 retry,
3550 Arc::clone(&self.sem),
3551 &ctx,
3552 &mut self.state,
3553 fallback_attempt,
3554 )
3555 .await;
3556 *seat = fallback_seat;
3557 *out = fallback_out;
3558 }
3559 }
3560 Ok(())
3561 }
3562
3563 async fn resume_unconfirmed_commands(
3587 &mut self,
3588 results: &mut [(usize, SeatState, AgentOutcome)],
3589 sent: &[SeatJob],
3590 prompts: &Prompts,
3591 run_id: &str,
3592 ) {
3593 for (wi, seat, out) in results.iter_mut() {
3594 let AgentOutcome::Ok(o) = &*out else {
3595 continue;
3596 };
3597 if !has_unconfirmed_command(&o.commands) {
3598 continue;
3599 }
3600 let Some(job) = sent.get(*wi) else { continue };
3601 if !has_context(&job.spec, seat, job.sessions) {
3602 self.state.event(
3603 "implement",
3604 format!(
3605 "{}: the reply named a command whose own CLI never confirmed the exit \
3606 status of, but there is no session left to resume",
3607 seat.key
3608 ),
3609 );
3610 continue;
3611 }
3612 self.state.event(
3613 "implement",
3614 format!(
3615 "{}: the reply named a command whose own CLI never confirmed the exit \
3616 status of; resuming the conversation",
3617 seat.key
3618 ),
3619 );
3620 let mut retry = job.clone();
3621 retry.seat = seat.clone();
3622 retry.prompt = prompt::resume_incomplete(
3623 "a command in your last reply had no confirmed exit status",
3624 );
3625 retry.timeout = retry_budget(job.timeout, true);
3626 retry.stem = format!("{}-confirm", job.stem);
3627 let cache = self.state.config.cache_dir();
3628 let ctx = WaveCtx {
3629 carry_seats: false,
3630 pauses: Some((&self.pause, &self.interrupt)),
3631 run: run_id,
3632 node: "implement",
3633 prompts,
3634 cache: cache.as_deref(),
3635 round: None,
3636 };
3637 let (resumed_seat, resumed) =
3638 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3639 *seat = resumed_seat;
3640 *out = resumed;
3641 }
3642 }
3643
3644 async fn continue_fix_report(
3665 &mut self,
3666 mut seat: SeatState,
3667 parse_err: String,
3668 job: &SeatJob,
3669 prompts: &Prompts,
3670 run_id: &str,
3671 round: usize,
3672 ) -> (
3673 SeatState,
3674 Option<FixReport>,
3675 Option<String>,
3676 ContinuationRecord,
3677 ) {
3678 let mut last_err = parse_err;
3679 let mut cumulative_wait_ms = 0u64;
3680 let mut attempts = 0usize;
3681 loop {
3682 if !has_context(&job.spec, &seat, job.sessions) {
3683 self.state.event(
3684 "fix",
3685 format!(
3686 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3687 session left to resume into"
3688 ),
3689 );
3690 let outcome = if attempts == 0 {
3691 ContinuationOutcome::NoSession
3692 } else {
3693 ContinuationOutcome::Exhausted
3694 };
3695 return (
3696 seat,
3697 None,
3698 Some(format!("unparsable fix report: {last_err}")),
3699 ContinuationRecord {
3700 attempts,
3701 cumulative_wait_ms,
3702 outcome,
3703 },
3704 );
3705 }
3706 if attempts >= MAX_FIX_CONTINUATIONS {
3707 self.state.event(
3708 "fix",
3709 format!(
3710 "round {round}: fixer's reply still had no adoption report after \
3711 {attempts} continuation(s) ({last_err}); giving up"
3712 ),
3713 );
3714 return (
3715 seat,
3716 None,
3717 Some(format!(
3718 "unparsable fix report after {attempts} continuation(s): {last_err}"
3719 )),
3720 ContinuationRecord {
3721 attempts,
3722 cumulative_wait_ms,
3723 outcome: ContinuationOutcome::Exhausted,
3724 },
3725 );
3726 }
3727 attempts += 1;
3728 self.state.event(
3729 "fix",
3730 format!(
3731 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3732 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3733 ),
3734 );
3735 let mut retry = job.clone();
3736 retry.seat = seat.clone();
3737 retry.prompt = prompt::resume_incomplete(&last_err);
3738 retry.timeout = retry_budget(job.timeout, true);
3739 retry.stem = format!("{}-continue{attempts}", job.stem);
3740 let cache = self.state.config.cache_dir();
3741 let ctx = WaveCtx {
3742 carry_seats: false,
3743 pauses: Some((&self.pause, &self.interrupt)),
3744 run: run_id,
3745 node: "fix",
3746 prompts,
3747 cache: cache.as_deref(),
3748 round: Some(round),
3749 };
3750 let (resumed_seat, resumed_out) = run_one(
3751 retry,
3752 Arc::clone(&self.sem),
3753 &ctx,
3754 &mut self.state,
3755 attempts,
3756 )
3757 .await;
3758 seat = resumed_seat;
3759 match resumed_out {
3760 AgentOutcome::Ok(o) => {
3761 cumulative_wait_ms += o.duration_ms;
3762 match verdict::extract_json::<FixReport>(&o.text) {
3763 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3764 self.state.event(
3765 "fix",
3766 format!(
3767 "round {round}: fixer's adoption report recovered after \
3768 {attempts} continuation(s)"
3769 ),
3770 );
3771 return (
3772 seat,
3773 Some(report),
3774 None,
3775 ContinuationRecord {
3776 attempts,
3777 cumulative_wait_ms,
3778 outcome: ContinuationOutcome::Resumed,
3779 },
3780 );
3781 }
3782 Ok(_) => {
3790 last_err = "the reply parsed, but it reported a command whose own CLI \
3791 never confirmed an exit status"
3792 .to_owned();
3793 }
3794 Err(e) => last_err = e.to_string(),
3795 }
3796 }
3797 AgentOutcome::Quota(o) => {
3798 cumulative_wait_ms += o.duration_ms;
3799 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3800 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3801 self.state.quota.push(loss);
3802 self.state.event(
3803 "fix",
3804 format!(
3805 "round {round}: continuation rate limited (quota); not retrying now"
3806 ),
3807 );
3808 return (
3809 seat,
3810 None,
3811 Some("rate limited (quota) while recovering the fix report".to_owned()),
3812 ContinuationRecord {
3813 attempts,
3814 cumulative_wait_ms,
3815 outcome: ContinuationOutcome::QuotaLost,
3816 },
3817 );
3818 }
3819 AgentOutcome::Dropped(o) => {
3820 cumulative_wait_ms += o.duration_ms;
3821 let why = o
3822 .dropped
3823 .as_ref()
3824 .map(|d| d.why.as_str())
3825 .unwrap_or("the CLI ended the stream without delivering its answer");
3826 last_err = format!("the CLI dropped the stream ({why})");
3827 }
3828 AgentOutcome::Failed(e) => last_err = e,
3829 AgentOutcome::ResourceBlocked(e) => {
3832 return (
3833 seat,
3834 None,
3835 Some(format!("the continuation was not started: {e}")),
3836 ContinuationRecord {
3837 attempts: attempts.saturating_sub(1),
3838 cumulative_wait_ms,
3839 outcome: ContinuationOutcome::NoSession,
3840 },
3841 );
3842 }
3843 }
3844 }
3845 }
3846
3847 fn after_implement(&mut self) -> Result<()> {
3848 if !self.state.candidate_preservation.is_empty() {
3849 bail!("candidate preservation incomplete; judging is unsafe");
3850 }
3851 if self.state.leaks.is_empty() {
3853 let cfg = self.state.config.blind.clone();
3854 let mut leaks = Vec::new();
3855 for c in &self.state.candidates {
3856 let Some(patch) =
3857 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3858 else {
3859 continue;
3860 };
3861 leaks.extend(blind::scan(
3862 &format!("candidate {} patch", c.label),
3863 &patch,
3864 &cfg.vendor_tokens,
3865 ));
3866 }
3867 if !leaks.is_empty() {
3868 let summary = leaks
3869 .iter()
3870 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3871 .collect::<Vec<_>>()
3872 .join(", ");
3873 match cfg.on_leak {
3874 LeakPolicy::Fail => {
3875 self.state.status = RunStatus::Failed;
3876 self.state
3877 .event("blind", format!("vendor text in a patch: {summary}"));
3878 self.state.leaks = leaks;
3879 self.state.save()?;
3880 self.settle_questions();
3881 bail!(
3882 "blind.on_leak = \"fail\" and vendor text reached a \
3883 judged patch: {summary}"
3884 );
3885 }
3886 LeakPolicy::Redact => self.state.event(
3887 "blind",
3888 format!("redacting vendor text for judging: {summary}"),
3889 ),
3890 LeakPolicy::Warn => self.state.event(
3891 "blind",
3892 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3893 ),
3894 }
3895 self.state.leaks = leaks;
3896 }
3897 }
3898
3899 if self.state.viable().is_empty() {
3900 if self.state.all_candidates_verified_noop() {
3901 self.state.status = RunStatus::VerifiedNoop;
3912 self.state.save()?;
3913 self.settle_questions();
3914 return Ok(());
3915 }
3916 self.state.status = RunStatus::Failed;
3917 self.state.save()?;
3918 self.settle_questions();
3919 bail!("no candidate produced a change; nothing to judge");
3920 }
3921 self.state.status = RunStatus::Judging;
3922 self.state.save()?;
3923 Ok(())
3924 }
3925
3926 async fn judge(&mut self) -> Result<()> {
3929 let run_id = self.state.id.clone();
3934 let prompts = self.state.config.prompts.clone();
3935 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3936 return Ok(());
3937 }
3938 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3939 if viable.len() == 1 {
3940 self.state.judge_skipped = true;
3947 self.state.event(
3948 "judge",
3949 format!(
3950 "only candidate {} produced a change; judging skipped",
3951 viable[0].label
3952 ),
3953 );
3954 self.state.save()?;
3955 return Ok(());
3956 }
3957 self.state.status = RunStatus::Judging;
3958
3959 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3960 let language = self.state.config.graph.language.clone();
3961 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3962 let sessions = self.state.config.graph.sessions;
3963 let artifacts = agent::artifacts_dir(&self.state.dir());
3964 let root = self.state.worktree_root();
3965 let base_short = short(&self.state.base_commit);
3966
3967 let mut jobs = Vec::new();
3968 let mut orders = Vec::new();
3969 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3970 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3971 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3972 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3973 let seat_key = format!("judge-{}", j + 1);
3974 let seat = self.seat(&seat_key, &spec.id);
3975 jobs.push(SeatJob {
3976 prompt: prompt::judge(
3977 &self.state.instruction,
3978 &views,
3979 self.roles.judges.len(),
3980 &base_short,
3981 &language,
3982 ),
3983 spec,
3984 seat,
3985 cwd: root.join(format!("judge-{}", j + 1)),
3986 timeout,
3987 allow_write: false,
3988 sessions,
3989 artifacts: artifacts.clone(),
3990 stem: format!("judge-{}", j + 1),
3991 handover: None,
3992 });
3993 }
3994
3995 self.state.event(
3996 "judge",
3997 format!(
3998 "{} judges ranking {} candidates blind",
3999 jobs.len(),
4000 viable.len()
4001 ),
4002 );
4003 let labels_for_check = labels.clone();
4004 let mut quota_losses = Vec::new();
4005 let cache = self.state.config.cache_dir();
4006 let ctx = WaveCtx {
4007 carry_seats: false,
4008 pauses: Some((&self.pause, &self.interrupt)),
4009 run: &run_id,
4010 node: "judge",
4011 prompts: &prompts,
4012 cache: cache.as_deref(),
4013 round: None,
4014 };
4015 let results = ask_json_wave::<Ranking>(
4016 jobs,
4017 Arc::clone(&self.sem),
4018 self.state.config.graph.retries,
4019 &self.roles.judge_roster,
4020 &ctx,
4021 &mut quota_losses,
4022 &mut self.state,
4023 &move |r: &Ranking| r.validate(&labels_for_check),
4024 )
4025 .await;
4026 if self.state.resource_block.is_some() {
4027 return self.stop_for_resource();
4028 }
4029 self.state.quota.extend(quota_losses);
4030
4031 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4032 let agent_id = seat.agent.clone();
4033 self.state.seats.insert(seat.key.clone(), seat);
4034 let mut record = Judgement {
4035 judge: j + 1,
4036 seat: format!("judge-{}", j + 1),
4037 agent: agent_id,
4038 ranking: Vec::new(),
4039 reasons: BTreeMap::new(),
4040 confidence: None,
4041 strengths: BTreeMap::new(),
4042 order: orders[j].clone(),
4043 failed: None,
4044 duration_ms: 0,
4045 };
4046 match res {
4047 Ok((ranking, out)) => {
4048 record.ranking = ranking.normalized();
4049 record.reasons = ranking.reasons;
4050 record.strengths = ranking.strengths;
4051 record.confidence = ranking.confidence;
4052 record.duration_ms = out.duration_ms;
4053 self.state.event(
4054 "judge",
4055 format!(
4056 "judge {} ranked {}",
4057 j + 1,
4058 record.ranking.iter().collect::<String>()
4059 ),
4060 );
4061 }
4062 Err(e) => {
4063 record.failed = Some(e.to_string());
4064 self.state
4065 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4066 }
4067 }
4068 self.state.judgements.push(record);
4069 self.state.save()?;
4070 }
4071 Ok(())
4072 }
4073
4074 async fn deliberate(&mut self) -> Result<()> {
4077 if self.hold_base_sync().await? {
4078 return Ok(());
4079 }
4080 let run_id = self.state.id.clone();
4085 let prompts = self.state.config.prompts.clone();
4086 let rounds = self.state.config.graph.deliberate_rounds;
4087 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4089 return Ok(());
4090 }
4091 let tops: Vec<char> = self
4092 .state
4093 .judgements
4094 .iter()
4095 .filter_map(|j| j.ranking.first().copied())
4096 .collect();
4097 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4098 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4099 self.state.event(
4100 "deliberate",
4101 format!("judges agreed on {} outright; no deliberation", tops[0]),
4102 );
4103 }
4104 self.state.status = RunStatus::Voting;
4105 self.state.save()?;
4106 return Ok(());
4107 }
4108
4109 self.state.status = RunStatus::Deliberating;
4110 self.state.event(
4111 "deliberate",
4112 format!(
4113 "split: first choices were {} — opening {rounds} round(s)",
4114 tops.iter().collect::<String>()
4115 ),
4116 );
4117
4118 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4119 let language = self.state.config.graph.language.clone();
4120 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4121 let sessions = self.state.config.graph.sessions;
4122 let artifacts = agent::artifacts_dir(&self.state.dir());
4123 let root = self.state.worktree_root();
4124 let base_short = short(&self.state.base_commit);
4125
4126 for round in self.state.deliberation.len() + 1..=rounds {
4130 let mut turns: Vec<DeliberationTurn> = Vec::new();
4131 let mut round_losses = Vec::new();
4132 let mut asked = 0;
4133 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4134 if self.state.judgements[j].failed.is_some() {
4135 continue;
4136 }
4137 let seat_key = format!("judge-{}", j + 1);
4138 let spec = self.occupant(&seat_key, spec);
4139 let mut seat = self.seat(&seat_key, &spec.id);
4140 let transcript = self.transcript(&turns, j);
4141 let build = |context: Option<&str>| {
4142 prompt::deliberate(
4143 &self.state.instruction,
4144 context,
4145 &transcript,
4146 round,
4147 rounds,
4148 &language,
4149 )
4150 };
4151 let block = self.candidate_block(&viable, &base_short);
4152 let full = build(Some(&block));
4153 let text = if has_context(&spec, &seat, sessions) {
4154 build(None)
4155 } else {
4156 full.clone()
4157 };
4158 let job = SeatJob {
4159 spec,
4160 seat: seat.clone(),
4161 prompt: text,
4162 cwd: root.join(format!("judge-{}", j + 1)),
4163 timeout,
4164 allow_write: false,
4165 sessions,
4166 artifacts: artifacts.clone(),
4167 stem: format!("delib-{round}-judge-{}", j + 1),
4168 handover: Some(full),
4169 };
4170 let cache = self.state.config.cache_dir();
4171 let ctx = WaveCtx {
4172 carry_seats: false,
4173 pauses: Some((&self.pause, &self.interrupt)),
4174 run: &run_id,
4175 node: "deliberate",
4176 prompts: &prompts,
4177 cache: cache.as_deref(),
4178 round: None,
4179 };
4180 let mut losses = Vec::new();
4184 asked += 1;
4185 let mut results = ask_wave_with::<String>(
4186 vec![job],
4187 Arc::clone(&self.sem),
4188 0,
4189 &self.roles.judge_roster,
4190 &ctx,
4191 &mut losses,
4192 &mut self.state,
4193 &|text: &str| {
4194 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4195 },
4196 )
4197 .await;
4198 if self.state.resource_block.is_some() {
4199 return self.stop_for_resource();
4200 }
4201 round_losses.extend(losses.iter().cloned());
4202 self.state.quota.extend(losses);
4203 let (updated, res, _) = results.pop().expect("one job in, one result out");
4204 seat = updated;
4205 let agent_id = seat.agent.clone();
4206 self.state.seats.insert(seat.key.clone(), seat);
4207 let body = match res {
4208 Ok((body, _)) => body,
4209 Err(e) => {
4212 self.state
4213 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4214 continue;
4215 }
4216 };
4217 let tentative = verdict::extract_json::<Position>(&body)
4218 .ok()
4219 .and_then(|p| p.tentative)
4220 .and_then(|s| s.trim().chars().next())
4221 .map(|c| c.to_ascii_uppercase());
4222 self.state.event(
4223 "deliberate",
4224 format!(
4225 "round {round}: judge {} now favours {}",
4226 j + 1,
4227 tentative.map_or("—".to_owned(), |c| c.to_string())
4228 ),
4229 );
4230 turns.push(DeliberationTurn {
4231 judge: j + 1,
4232 agent: agent_id,
4233 body: blind::sanitize_prose(&body, &self.state.config.blind),
4234 tentative,
4235 });
4236 }
4237 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4238 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4239 return self.stop_for_resource();
4240 }
4241 self.state
4242 .deliberation
4243 .push(DeliberationRound { round, turns });
4244 self.state.save()?;
4245 }
4246
4247 self.state.status = RunStatus::Voting;
4248 self.state.save()?;
4249 Ok(())
4250 }
4251
4252 async fn vote(&mut self) -> Result<()> {
4255 if self.hold_base_sync().await? {
4256 return Ok(());
4257 }
4258 let run_id = self.state.id.clone();
4263 let prompts = self.state.config.prompts.clone();
4264 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4265 return Ok(());
4266 }
4267 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4268 if viable.len() == 1 {
4269 return Ok(());
4270 }
4271 self.state.status = RunStatus::Voting;
4272
4273 let language = self.state.config.graph.language.clone();
4274 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4275 let sessions = self.state.config.graph.sessions;
4276 let artifacts = agent::artifacts_dir(&self.state.dir());
4277 let root = self.state.worktree_root();
4278 let base_short = short(&self.state.base_commit);
4279 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4280
4281 let mut jobs = Vec::new();
4282 let mut seats_at = Vec::new();
4283 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4284 if self
4285 .state
4286 .judgements
4287 .get(j)
4288 .is_some_and(|r| r.failed.is_some())
4289 {
4290 continue;
4291 }
4292 let seat_key = format!("judge-{}", j + 1);
4293 let spec = self.occupant(&seat_key, spec);
4294 let seat = self.seat(&seat_key, &spec.id);
4295 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4296 let text = if has_context(&spec, &seat, sessions) {
4297 prompt::final_vote(&viable, &language)
4298 } else {
4299 full.clone()
4300 };
4301 jobs.push(SeatJob {
4302 spec,
4303 seat,
4304 prompt: text,
4305 cwd: root.join(format!("judge-{}", j + 1)),
4306 timeout,
4307 allow_write: false,
4308 sessions,
4309 artifacts: artifacts.clone(),
4310 stem: format!("vote-judge-{}", j + 1),
4311 handover: Some(full),
4312 });
4313 seats_at.push(j);
4314 }
4315
4316 self.state.event(
4317 "vote",
4318 format!(
4319 "collecting {} final votes one by one, privately",
4320 jobs.len()
4321 ),
4322 );
4323 let allowed = viable.clone();
4324 let mut quota_losses = Vec::new();
4325 let cache = self.state.config.cache_dir();
4326 let ctx = WaveCtx {
4327 carry_seats: false,
4328 pauses: Some((&self.pause, &self.interrupt)),
4329 run: &run_id,
4330 node: "vote",
4331 prompts: &prompts,
4332 cache: cache.as_deref(),
4333 round: None,
4334 };
4335 let results = ask_json_wave::<FinalVote>(
4336 jobs,
4337 Arc::clone(&self.sem),
4338 self.state.config.graph.retries,
4339 &self.roles.judge_roster,
4340 &ctx,
4341 &mut quota_losses,
4342 &mut self.state,
4343 &move |v: &FinalVote| match v.label() {
4344 Some(c) if allowed.contains(&c) => Ok(()),
4345 other => bail!("vote {other:?} is not one of {allowed:?}"),
4346 },
4347 )
4348 .await;
4349 if self.state.resource_block.is_some() {
4350 return self.stop_for_resource();
4351 }
4352 self.state.quota.extend(quota_losses);
4353
4354 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4355 let agent_id = seat.agent.clone();
4356 self.state.seats.insert(seat.key.clone(), seat);
4357 let initial = self
4358 .state
4359 .judgements
4360 .get(j)
4361 .and_then(|r| r.ranking.first().copied());
4362 let mut record = VoteRecord {
4363 judge: j + 1,
4364 agent: agent_id,
4365 vote: None,
4366 reason: String::new(),
4367 changed: false,
4368 };
4369 match res {
4370 Ok((v, _)) => {
4371 record.vote = v.label();
4372 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4373 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4374 self.state.event(
4375 "vote",
4376 format!(
4377 "judge {} voted {}{}",
4378 j + 1,
4379 record.vote.unwrap_or('?'),
4380 if record.changed { " (changed)" } else { "" }
4381 ),
4382 );
4383 }
4384 Err(e) => {
4385 self.state
4386 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4387 }
4388 }
4389 self.state.votes.push(record);
4390 self.state.save()?;
4391 }
4392 Ok(())
4393 }
4394
4395 fn tally(&mut self) -> Result<()> {
4398 if self.state.tally.is_some() {
4399 return Ok(());
4400 }
4401 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4402 let tops: Vec<char> = self
4403 .state
4404 .judgements
4405 .iter()
4406 .filter_map(|j| j.ranking.first().copied())
4407 .collect();
4408 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4409
4410 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4413 let mut cast: Vec<char> = Vec::new();
4414 for (i, j) in self.state.judgements.iter().enumerate() {
4415 let vote = self
4416 .state
4417 .votes
4418 .iter()
4419 .find(|v| v.judge == i + 1)
4420 .and_then(|v| v.vote)
4421 .or_else(|| j.ranking.first().copied());
4422 if let Some(v) = vote {
4423 *first_choice.entry(v).or_insert(0) += 1;
4424 cast.push(v);
4425 }
4426 }
4427
4428 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4429 for j in &self.state.judgements {
4430 let n = j.ranking.len();
4431 for (pos, label) in j.ranking.iter().enumerate() {
4432 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4433 }
4434 }
4435
4436 let best = first_choice.values().copied().max().unwrap_or(0);
4437 let mut leaders: Vec<char> = first_choice
4438 .iter()
4439 .filter(|(_, v)| **v == best)
4440 .map(|(k, _)| *k)
4441 .collect();
4442 let mut tie_break = None;
4443 if leaders.len() > 1 {
4444 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4445 let borda_leaders: Vec<char> = leaders
4446 .iter()
4447 .copied()
4448 .filter(|l| borda[l] == top_borda)
4449 .collect();
4450 tie_break = Some(if borda_leaders.len() == 1 {
4451 format!(
4452 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4453 leaders.len()
4454 )
4455 } else {
4456 format!(
4457 "{} way tie on both first-choice votes and Borda points, broken by label order",
4458 leaders.len()
4459 )
4460 });
4461 leaders = borda_leaders;
4462 leaders.sort_unstable();
4463 }
4464 let winner = *leaders
4465 .first()
4466 .or(viable.first())
4467 .context("no candidate to declare a winner from")?;
4468
4469 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4470 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4471 let deliberated = !self.state.deliberation.is_empty();
4472
4473 let quota_seats: std::collections::BTreeSet<&str> =
4477 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4478 let mut present = 0usize;
4479 for (i, j) in self.state.judgements.iter().enumerate() {
4480 if quota_seats.contains(j.seat.as_str()) {
4481 continue;
4482 }
4483 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4484 let voted = self
4485 .state
4486 .votes
4487 .iter()
4488 .any(|v| v.judge == i + 1 && v.vote.is_some());
4489 if ranked || voted {
4490 present += 1;
4491 }
4492 }
4493 let needs_quorum = viable.len() > 1;
4499 let judges_total = if needs_quorum {
4500 self.roles.judges.len()
4501 } else {
4502 0
4503 };
4504 let quorum = if needs_quorum {
4505 judges_total / 2 + 1
4506 } else {
4507 0
4508 };
4509 let met_quorum = !needs_quorum || present >= quorum;
4510 let uncontested = (!needs_quorum).then(|| {
4511 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4512 });
4513
4514 self.state.event(
4515 "tally",
4516 match &uncontested {
4517 Some(reason) => format!("winner {winner} — {reason}"),
4518 None => format!(
4519 "winner {winner} — votes {} | initial {} | {} changed | \
4520 {present}/{judges_total} judges{}",
4521 first_choice
4522 .iter()
4523 .map(|(k, v)| format!("{k}:{v}"))
4524 .collect::<Vec<_>>()
4525 .join(" "),
4526 if unanimous_initial {
4527 "unanimous"
4528 } else {
4529 "split"
4530 },
4531 changed_votes,
4532 if met_quorum {
4533 String::new()
4534 } else {
4535 format!(" — below quorum ({quorum} required)")
4536 },
4537 ),
4538 },
4539 );
4540 if !met_quorum {
4541 self.state.event(
4542 "stall",
4543 format!(
4544 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4545 the run stops here, resumable"
4546 ),
4547 );
4548 }
4549 self.state.tally = Some(Tally {
4550 first_choice,
4551 borda,
4552 winner,
4553 rankings: tops.len(),
4554 unanimous_initial,
4555 deliberated,
4556 changed_votes,
4557 unanimous_final,
4558 tie_break,
4559 judges: judges_total,
4560 present,
4561 quorum,
4562 met_quorum,
4563 uncontested,
4564 });
4565 self.state.status = if met_quorum {
4566 RunStatus::Reviewing
4567 } else {
4568 RunStatus::Stalled
4569 };
4570 self.state.save()?;
4571 Ok(())
4572 }
4573
4574 #[allow(clippy::too_many_lines)]
4595 async fn recover_stall(&mut self) -> Result<bool> {
4596 let run_id = self.state.id.clone();
4601 let prompts = self.state.config.prompts.clone();
4602 let quota_seats: BTreeSet<&str> =
4607 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4608 let absent: Vec<String> = self
4609 .state
4610 .judgements
4611 .iter()
4612 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4613 .map(|j| j.seat.clone())
4614 .collect();
4615 if absent.is_empty() {
4616 return Ok(false);
4617 }
4618 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4619 if viable.len() <= 1 {
4620 return Ok(false);
4621 }
4622 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4623 let language = self.state.config.graph.language.clone();
4624 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4625 let sessions = self.state.config.graph.sessions;
4626 let artifacts = agent::artifacts_dir(&self.state.dir());
4627 let root = self.state.worktree_root();
4628 let base_short = short(&self.state.base_commit);
4629 let candidates: Vec<Candidate> = viable.clone();
4630
4631 let mut positions: Vec<usize> = absent
4633 .iter()
4634 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4635 .collect();
4636 if positions.is_empty() {
4637 return Ok(false);
4638 }
4639 positions.sort_unstable();
4640 positions.dedup();
4641
4642 let mut judge_jobs = Vec::new();
4644 for &j in &positions {
4645 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4646 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4647 let seat_key = format!("judge-{}", j + 1);
4648 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4649 let seat = self.seat(&seat_key, &spec.id);
4650 judge_jobs.push(SeatJob {
4651 spec,
4652 seat,
4653 prompt: prompt::judge(
4654 &self.state.instruction,
4655 &views,
4656 self.roles.judges.len(),
4657 &base_short,
4658 &language,
4659 ),
4660 cwd: root.join(seat_key),
4661 timeout,
4662 allow_write: false,
4663 sessions,
4664 artifacts: artifacts.clone(),
4665 stem: format!("judge-{}-recover", j + 1),
4666 handover: None,
4667 });
4668 }
4669
4670 let labels_for_check = labels.clone();
4671 let mut judge_losses = Vec::new();
4672 let retries = self.state.config.graph.retries;
4673 let cache = self.state.config.cache_dir();
4674 let ctx = WaveCtx {
4675 carry_seats: false,
4676 pauses: Some((&self.pause, &self.interrupt)),
4677 run: &run_id,
4678 node: "judge",
4679 prompts: &prompts,
4680 cache: cache.as_deref(),
4681 round: None,
4682 };
4683 let results = ask_json_wave::<Ranking>(
4684 judge_jobs,
4685 Arc::clone(&self.sem),
4686 retries,
4687 &self.roles.judge_roster,
4688 &ctx,
4689 &mut judge_losses,
4690 &mut self.state,
4691 &move |r: &Ranking| r.validate(&labels_for_check),
4692 )
4693 .await;
4694 if self.state.resource_block.is_some() {
4695 self.stop_for_resource()?;
4696 return Ok(false);
4697 }
4698
4699 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4701 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4702 let agent_id = seat.agent.clone();
4703 self.state.seats.insert(seat.key.clone(), seat);
4704 let record = &mut self.state.judgements[j];
4705 match res {
4706 Ok((ranking, out)) => {
4707 record.agent = agent_id;
4708 record.ranking = ranking.normalized();
4709 record.reasons = ranking.reasons;
4710 record.strengths = ranking.strengths;
4711 record.confidence = ranking.confidence;
4712 record.failed = None;
4713 record.duration_ms = out.duration_ms;
4714 recovered.insert(j);
4715 self.state.event(
4716 "recover",
4717 format!("judge {} ranked again after the limit", j + 1),
4718 );
4719 }
4720 Err(e) => {
4721 self.state
4722 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4723 }
4724 }
4725 }
4726
4727 let mut vote_jobs = Vec::new();
4729 let mut vote_pos: Vec<usize> = Vec::new();
4730 for &j in &recovered {
4731 let seat_key = format!("judge-{}", j + 1);
4732 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4733 let seat = self.seat(&seat_key, &spec.id);
4734 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4735 let text = if has_context(&spec, &seat, sessions) {
4736 prompt::final_vote(&labels, &language)
4737 } else {
4738 full.clone()
4739 };
4740 vote_jobs.push(SeatJob {
4741 spec,
4742 seat,
4743 prompt: text,
4744 cwd: root.join(seat_key),
4745 timeout,
4746 allow_write: false,
4747 sessions,
4748 artifacts: artifacts.clone(),
4749 stem: format!("vote-judge-{}-recover", j + 1),
4750 handover: Some(full),
4751 });
4752 vote_pos.push(j);
4753 }
4754 let allowed = labels.clone();
4755 let mut vote_losses = Vec::new();
4756 let vote_retries = self.state.config.graph.retries;
4757 let vote_cache = self.state.config.cache_dir();
4758 let ctx = WaveCtx {
4759 carry_seats: false,
4760 pauses: Some((&self.pause, &self.interrupt)),
4761 run: &run_id,
4762 node: "vote",
4763 prompts: &prompts,
4764 cache: vote_cache.as_deref(),
4765 round: None,
4766 };
4767 let votes = ask_json_wave::<FinalVote>(
4768 vote_jobs,
4769 Arc::clone(&self.sem),
4770 vote_retries,
4771 &self.roles.judge_roster,
4772 &ctx,
4773 &mut vote_losses,
4774 &mut self.state,
4775 &move |v: &FinalVote| match v.label() {
4776 Some(c) if allowed.contains(&c) => Ok(()),
4777 other => bail!("vote {other:?} is not one of {allowed:?}"),
4778 },
4779 )
4780 .await;
4781 if self.state.resource_block.is_some() {
4782 self.stop_for_resource()?;
4783 return Ok(false);
4784 }
4785 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4786 let agent_id = seat.agent.clone();
4787 self.state.seats.insert(seat.key.clone(), seat);
4788 match res {
4789 Ok((v, _)) => {
4790 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4791 rec.vote = v.label();
4792 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4793 } else {
4794 self.state.votes.push(VoteRecord {
4795 judge: j + 1,
4796 agent: agent_id,
4797 vote: v.label(),
4798 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4799 changed: false,
4800 });
4801 }
4802 self.state.event(
4803 "recover",
4804 format!("judge {} voted again after the limit", j + 1),
4805 );
4806 }
4807 Err(e) => {
4808 self.state
4809 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4810 }
4811 }
4812 }
4813
4814 let recovered_keys: BTreeSet<String> = recovered
4818 .iter()
4819 .map(|&j| format!("judge-{}", j + 1))
4820 .collect();
4821 self.state
4822 .quota
4823 .retain(|q| !recovered_keys.contains(&q.seat));
4824 for loss in judge_losses.into_iter().chain(vote_losses) {
4828 if recovered_keys.contains(&loss.seat) {
4829 continue;
4830 }
4831 self.state.quota.retain(|q| q.seat != loss.seat);
4832 self.state.quota.push(loss);
4833 }
4834
4835 self.state.tally = None;
4837 self.tally()?;
4838 Ok(self
4839 .state
4840 .tally
4841 .as_ref()
4842 .map(|t| t.met_quorum)
4843 .unwrap_or(false))
4844 }
4845
4846 async fn graft(&mut self) -> Result<()> {
4862 let Err(e) = self.graft_inner().await else {
4863 return Ok(());
4864 };
4865 if e.is::<crate::repo_lock::ResourceBlocked>() {
4866 return Err(e);
4867 }
4868 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4869 self.state.status = RunStatus::Blocked;
4870 self.state.event("graft", format!("blocked: {e:#}"));
4871 self.state.save()?;
4872 return Err(e);
4873 }
4874 if let Some(rec) = self.state.graft.clone()
4879 && !rec.outcome.settled()
4880 {
4881 if let Some(winner) = self.state.winner().cloned() {
4882 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4883 if re.is::<crate::repo_lock::ResourceBlocked>() {
4884 return Err(re);
4885 }
4886 self.state.status = RunStatus::Blocked;
4887 self.state.event("graft", format!("blocked: {re:#}"));
4888 self.state.save()?;
4889 return Err(re);
4890 }
4891 }
4892 if let Some(g) = self.state.graft.as_mut() {
4893 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4894 }
4895 }
4896 self.state.event("graft", format!("skipped: {e:#}"));
4897 if let Err(e) = self.state.save() {
4898 tracing::warn!("save after graft: {e:#}");
4899 }
4900 Ok(())
4901 }
4902
4903 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4906 let _coordination = self.coordinate(wt).await?;
4907 crate::recovery::restore_repair(wt, pre)
4908 .await
4909 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4910 }
4911
4912 fn graft_skipped(&mut self, why: &str) {
4914 let message = format!("skipped: {why}");
4915 if !self
4916 .state
4917 .events
4918 .iter()
4919 .any(|e| e.node == "graft" && e.message == message)
4920 {
4921 self.state.event("graft", message);
4922 }
4923 }
4924
4925 async fn graft_inner(&mut self) -> Result<()> {
4926 if let Some(rec) = self.state.graft.clone()
4929 && !rec.outcome.settled()
4930 {
4931 if self.state.reviews.is_empty()
4935 && let Some(winner) = self.state.winner().cloned()
4936 {
4937 self.graft_restore(&winner.worktree, &rec.pre).await?;
4938 }
4939 if rec.outcome == GraftOutcome::WaitingForRestore {
4940 self.state.graft = None;
4941 self.state.save()?;
4942 } else {
4943 if let Some(g) = self.state.graft.as_mut() {
4944 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4945 }
4946 self.state.event(
4947 "graft",
4948 "rolled back: the previous round was interrupted; the winner is untouched",
4949 );
4950 self.state.save()?;
4951 return Ok(());
4952 }
4953 }
4954 let plan = match graft::plan(&self.state) {
4955 Ok(plan) => plan,
4956 Err(skip) => {
4957 if skip != graft::Skip::AlreadyDone {
4958 self.graft_skipped(skip.reason());
4959 self.state.save()?;
4960 }
4961 return Ok(());
4962 }
4963 };
4964 let Some(winner) = self.state.winner().cloned() else {
4965 return Ok(());
4966 };
4967 let wt = winner.worktree.clone();
4968 if !git::is_clean(&wt).await? {
4969 self.graft_skipped("the winner's worktree has uncommitted changes");
4970 self.state.save()?;
4971 return Ok(());
4972 }
4973 let pre = git::rev_parse(&wt, "HEAD").await?;
4974 self.state.graft = Some(GraftRecord {
4977 pre: pre.clone(),
4978 post: None,
4979 lines: None,
4980 limit: None,
4981 agent: None,
4982 outcome: GraftOutcome::Running,
4983 requested: graft::requested(&plan),
4984 adoption: None,
4985 adoption_error: None,
4986 });
4987 self.state.event(
4988 "graft",
4989 format!(
4990 "porting strengths of candidate(s) {} onto {}",
4991 plan.strengths
4992 .iter()
4993 .map(|(l, _, _)| l.to_string())
4994 .collect::<Vec<_>>()
4995 .join(", "),
4996 plan.winner
4997 ),
4998 );
4999 self.state.save()?;
5000
5001 let fix_prompt = prompt::graft(
5002 &self.state.instruction,
5003 plan.winner,
5004 &plan.strengths,
5005 &self.state.config.graph.language,
5006 );
5007 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5008 let sessions = self.state.config.graph.sessions;
5009 let artifacts = agent::artifacts_dir(&self.state.dir());
5010 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5011 let (_, seat, out) = self
5012 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5013 SeatJob {
5014 prompt: fix_prompt.clone(),
5015 spec,
5016 seat,
5017 cwd: wt.clone(),
5018 timeout,
5019 allow_write: true,
5020 sessions,
5021 artifacts: artifacts.clone(),
5022 stem: "graft".to_owned(),
5023 handover: None,
5024 }
5025 })
5026 .await;
5027 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5028 self.state.graft.as_mut().expect("graft record").outcome =
5031 GraftOutcome::WaitingForRestore;
5032 return self.stop_for_resource();
5033 }
5034 let mut why: Option<String> = None;
5035 match &out {
5036 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5037 why = Some("the fixer produced nothing usable".to_owned());
5038 }
5039 AgentOutcome::Ok(_) => {}
5040 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5041 AgentOutcome::Quota(o) => {
5042 self.state.quota.push(QuotaLoss::from_report(
5043 seat.key.clone(),
5044 "graft",
5045 o.quota.as_ref(),
5046 ));
5047 why = Some("rate limited (quota)".to_owned());
5048 }
5049 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5050 why = Some(format!(
5051 "the fixer failed: {}",
5052 e.lines()
5053 .next()
5054 .unwrap_or("")
5055 .chars()
5056 .take(160)
5057 .collect::<String>()
5058 ));
5059 }
5060 }
5061 let agent_id = seat.agent.clone();
5062 self.state.seats.insert(seat.key.clone(), seat);
5063
5064 let mut post = None;
5065 let mut lines = None;
5066 let _coordination = self.coordinate(&wt).await?;
5067 if why.is_none() {
5068 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5069 if let Ok(r) = &rescued {
5070 self.state.note_withheld("graft", &r.withheld);
5071 }
5072 let head = git::rev_parse(&wt, "HEAD").await?;
5073 let dirty = match &rescued {
5077 Ok(r) => git::status_porcelain(&wt)
5078 .await
5079 .map(|st| {
5080 st.lines().any(|l| {
5081 !r.withheld
5082 .iter()
5083 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5084 })
5085 })
5086 .unwrap_or(true),
5087 Err(_) => true,
5088 };
5089 if dirty {
5090 why = Some("the fixer left changes that could not be committed".to_owned());
5091 } else if head == pre {
5092 why = Some("the fixer changed nothing".to_owned());
5093 } else {
5094 lines = git::changed_lines(&wt, &pre, &head).await;
5095 post = Some(head);
5096 }
5097 }
5098 let (adoption, adoption_error) = match &out {
5099 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5100 _ => (None, None),
5101 };
5102 let requested = graft::requested(&plan);
5103 let outcome = match why {
5104 None => {
5105 self.state.event(
5106 "graft",
5107 format!(
5108 "applied: {}",
5109 lines.map_or_else(
5110 || "changed lines unavailable".to_owned(),
5111 |n| format!("{n} changed lines")
5112 )
5113 ),
5114 );
5115 GraftOutcome::Applied
5116 }
5117 Some(why) => {
5118 crate::recovery::restore_repair(&wt, &pre)
5121 .await
5122 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5123 self.state.event(
5124 "graft",
5125 format!("rolled back: {why}; the winner is untouched"),
5126 );
5127 GraftOutcome::RolledBack(why)
5128 }
5129 };
5130 self.state.graft = Some(GraftRecord {
5131 pre,
5132 post,
5133 lines,
5134 limit: None,
5135 agent: Some(agent_id),
5136 outcome,
5137 requested,
5138 adoption,
5139 adoption_error,
5140 });
5141 self.state.save()?;
5142 Ok(())
5143 }
5144
5145 async fn fold_losers(&mut self) -> Result<()> {
5148 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5149 return Ok(());
5150 };
5151 if !self.state.candidate_preservation.is_empty() {
5152 bail!("candidate preservation incomplete; refusing cleanup");
5153 }
5154 let repo = self.state.repo.clone();
5155 let _coordination = self.coordinate(&repo).await?;
5156 let mut folded = Vec::new();
5157 for i in 0..self.state.candidates.len() {
5158 let c = &self.state.candidates[i];
5159 if c.label == winner || c.folded {
5160 continue;
5161 }
5162 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5163 remove_preserved_worktree(&repo, &wt).await?;
5164 if git::branch_exists(&repo, &branch).await?
5165 && !git::branch_delete(&repo, &branch).await?
5166 {
5167 bail!("candidate branch removal refused");
5168 }
5169 self.state.candidates[i].folded = true;
5170 self.state.save()?;
5171 folded.push(label.to_string());
5172 }
5173 let root = self.state.worktree_root();
5175 for j in 1..=self.roles.judges.len() {
5176 let wt = root.join(format!("judge-{j}"));
5177 if wt.exists() {
5178 remove_preserved_worktree(&repo, &wt).await?;
5179 }
5180 }
5181 if self.state.config.graph.advise {
5184 for k in 1..=self.state.config.graph.advisors {
5185 let wt = root.join(format!("advisor-{k}"));
5186 if wt.exists() {
5187 remove_preserved_worktree(&repo, &wt).await?;
5188 }
5189 }
5190 }
5191 if !folded.is_empty() {
5192 self.state
5193 .event("fold", format!("folded candidates {}", folded.join(", ")));
5194 self.state.save()?;
5195 }
5196 Ok(())
5197 }
5198
5199 async fn hold_base_sync(&mut self) -> Result<bool> {
5202 self.hold_base_sync_as(None).await
5203 }
5204
5205 async fn hold_base_sync_kind(
5206 &mut self,
5207 kind: crate::base_sync_recovery::StopKind,
5208 ) -> Result<()> {
5209 self.hold_base_sync_as(Some(kind)).await?;
5210 Ok(())
5211 }
5212
5213 async fn hold_base_sync_as(
5214 &mut self,
5215 kind: Option<crate::base_sync_recovery::StopKind>,
5216 ) -> Result<bool> {
5217 if !self.state.base_sync_blocked() {
5218 self.state.base_sync_recovery.stop = None;
5219 return Ok(false);
5220 }
5221 let head = match self.state.winner() {
5222 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5223 .await
5224 .unwrap_or_default(),
5225 None => String::new(),
5226 };
5227 self.state.record_base_sync_stop(&head);
5228 if let Some(kind) = kind {
5229 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
5230 }
5231 self.state.save()?;
5232 Ok(true)
5233 }
5234
5235 async fn sync_to_base(&mut self) -> Result<()> {
5277 if self.state.status == RunStatus::AlreadyInBase {
5278 return Ok(());
5279 }
5280 self.hold_base_sync().await?;
5281 let conflicted = self
5282 .state
5283 .base_sync
5284 .as_ref()
5285 .is_some_and(|s| s.conflict.is_some());
5286 let Some(winner) = self.state.winner().cloned() else {
5287 return Ok(());
5288 };
5289
5290 let repo = self.state.repo.clone();
5291 let remote = self.state.config.merge.remote.clone();
5292 let base_branch = self.state.base_branch.clone();
5293 let tracking = format!("{remote}/{base_branch}");
5294
5295 let fetched_base =
5296 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5297 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
5302 return Ok(());
5303 };
5304
5305 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5306 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5307 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5308
5309 if (behind > 0 || conflicted || head == tip)
5318 && self
5319 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5320 .await?
5321 {
5322 return Ok(());
5323 }
5324 if conflicted && self.state.base_recovery_pending() {
5325 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5326 let owner =
5327 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5328 if !fetched_base
5329 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5330 || !git::is_clean(&winner.worktree).await?
5331 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5332 {
5333 return Ok(());
5334 }
5335 }
5336 let recovery = conflicted && self.state.consume_base_recovery(&head);
5337 if conflicted {
5338 self.state.save()?;
5340 if !recovery {
5341 return Ok(());
5342 }
5343 }
5344
5345 if behind == 0 {
5346 if let Some(from) = self
5355 .state
5356 .rebase_fixes
5357 .iter()
5358 .rev()
5359 .find_map(|r| r.from.clone())
5360 && from != head
5361 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5362 .await
5363 .is_ok_and(|o| o.ok())
5364 {
5365 git::sync_to_head(&winner.worktree).await?;
5366 }
5367 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5373 if let Some(why) = &conflict {
5374 self.state.status = RunStatus::Blocked;
5375 self.state.event("land", why.clone());
5376 }
5377 self.state.base_sync = Some(BaseSync {
5378 tip,
5379 behind: 0,
5380 attempts: attempts + usize::from(recovery),
5381 conflict,
5382 already_in: None,
5383 });
5384 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5385 .await?;
5386 self.state.save()?;
5387 return Ok(());
5388 }
5389
5390 if attempts >= BASE_SYNC_ROUNDS && !recovery {
5391 let why = format!(
5392 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5393 synchronization(s); synchronizing again would only race it",
5394 winner.branch
5395 );
5396 self.state.status = RunStatus::Blocked;
5397 self.state.base_sync = Some(BaseSync {
5398 tip,
5399 behind,
5400 attempts,
5401 conflict: Some(why.clone()),
5402 already_in: None,
5403 });
5404 self.state.event("land", why);
5405 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5406 .await?;
5407 self.state.save()?;
5408 return Ok(());
5409 }
5410
5411 if self.state.foreign_checkout.is_some() {
5412 let wait = repository_wait(&self.state, None);
5415 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5416 let merged = git::git_raw(
5417 &winner.worktree,
5418 &[
5419 "-c",
5420 "core.hooksPath=/dev/null",
5421 "merge",
5422 "--no-ff",
5423 "--no-autostash",
5424 "--no-edit",
5425 "-m",
5426 &format!("Merge {tracking} before verifying"),
5427 &tip,
5428 ],
5429 )
5430 .await?;
5431 let conflict = if merged.ok() {
5432 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5433 self.state.event(
5434 "land",
5435 format!("merged {tracking} into {} before verifying", winner.branch),
5436 );
5437 None
5438 } else {
5439 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5440 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5441 }
5442 let why = format!(
5443 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5444 winner.branch,
5445 merged.stderr.chars().take(600).collect::<String>()
5446 );
5447 self.state.status = RunStatus::Blocked;
5448 self.state.event("land", &why);
5449 Some(why)
5450 };
5451 let synced = conflict.is_none();
5452 self.state.base_sync = Some(BaseSync {
5453 tip,
5454 behind: if synced { 0 } else { behind },
5455 attempts: attempts + 1,
5456 conflict,
5457 already_in: None,
5458 });
5459 self.state.save()?;
5460 return Ok(());
5461 }
5462
5463 self.state.event(
5464 "land",
5465 format!(
5466 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5467 winner.branch
5468 ),
5469 );
5470 self.state.save()?;
5471
5472 let branch_tracking = format!("{remote}/{}", winner.branch);
5478 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5479 let remote_tip = if self.state.task_pr_branch().is_none()
5480 && matches!(&fetched_branch, Ok(o) if o.ok())
5481 {
5482 git::rev_parse(&repo, &branch_tracking).await.ok()
5483 } else {
5484 None
5485 };
5486 if let Some(theirs) = &remote_tip
5490 && !git::is_ancestor(&repo, theirs, &head).await
5491 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5492 .await
5493 .is_ok_and(|missing| missing.is_empty())
5494 {
5495 let why = format!(
5496 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5497 them",
5498 short(theirs),
5499 winner.branch
5500 );
5501 self.state.status = RunStatus::Blocked;
5502 self.state.base_sync = Some(BaseSync {
5503 tip,
5504 behind,
5505 attempts: attempts + usize::from(recovery),
5506 conflict: Some(why.clone()),
5507 already_in: None,
5508 });
5509 self.state.event("land", why);
5510 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5511 .await?;
5512 self.state.save()?;
5513 return Ok(());
5514 }
5515
5516 let scratch = self.state.dir().join("base-sync");
5517 let rebased =
5518 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5519 .await
5520 {
5521 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5522 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5523 Err(e) => Err(e),
5524 };
5525 let attempts = attempts + 1;
5526 let stop_kind = match &rebased {
5527 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5528 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5529 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5530 };
5531 match rebased {
5532 Ok(None) => {
5533 git::sync_to_head(&winner.worktree).await?;
5537 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5538 let mut conflict = None;
5539 if let Some(pinned) = &remote_tip {
5540 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5541 match pushed {
5542 Ok(o) if o.ok() => self.state.event(
5543 "land",
5544 format!("pushed rebased {} to {remote}", winner.branch),
5545 ),
5546 Ok(o) => {
5547 conflict = Some(format!(
5548 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5549 winner.branch,
5550 short(pinned),
5551 o.stderr.chars().take(600).collect::<String>()
5552 ));
5553 }
5554 Err(e) => {
5555 conflict = Some(format!(
5556 "rebased {} locally but could not push it: {e:#}",
5557 winner.branch
5558 ));
5559 }
5560 }
5561 }
5562 if let Some(why) = &conflict {
5563 self.state.status = RunStatus::Blocked;
5564 self.state.event("land", why.clone());
5565 }
5566 self.state.base_sync = Some(BaseSync {
5567 tip: tip.clone(),
5568 behind: 0,
5569 attempts,
5570 conflict,
5571 already_in: None,
5572 });
5573 self.state
5574 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5575 }
5576 Ok(Some(conflict)) => {
5577 let why = format!(
5578 "{} conflicts with {tracking} and did not rebase: {}",
5579 winner.branch,
5580 conflict.chars().take(600).collect::<String>()
5581 );
5582 self.state.status = RunStatus::Blocked;
5583 self.state.base_sync = Some(BaseSync {
5584 tip,
5585 behind,
5586 attempts,
5587 conflict: Some(why.clone()),
5588 already_in: None,
5589 });
5590 self.state.event("land", why);
5591 }
5592 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5596 self.state.save()?;
5597 return Ok(());
5598 }
5599 Err(e) => {
5600 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5601 self.state.status = RunStatus::Blocked;
5602 self.state.base_sync = Some(BaseSync {
5603 tip,
5604 behind,
5605 attempts,
5606 conflict: Some(why.clone()),
5607 already_in: None,
5608 });
5609 self.state.event("land", why);
5610 }
5611 }
5612 if !self.state.base_sync_blocked() {
5613 self.state.base_sync_recovery.stop = None;
5614 }
5615 self.hold_base_sync_kind(stop_kind).await?;
5616 self.state.save()?;
5617 Ok(())
5618 }
5619
5620 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5626 if self.state.task_pr_branch().is_some() {
5629 return None;
5630 }
5631 let repo = self.state.repo.clone();
5632 let remote = self.state.config.merge.remote.clone();
5633 let fetched = git::fetch(&repo, &remote, branch).await;
5634 if !matches!(&fetched, Ok(o) if o.ok()) {
5635 return None;
5636 }
5637 let branch_tracking = format!("{remote}/{branch}");
5638 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5639 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5640 return None;
5641 }
5642 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5643 .await
5644 .is_ok_and(|missing| missing.is_empty())
5645 {
5646 return Some(format!(
5647 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5648 them",
5649 short(&theirs)
5650 ));
5651 }
5652 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5653 Ok(o) if o.ok() => {
5654 self.state
5655 .event("land", format!("pushed rebased {branch} to {remote}"));
5656 None
5657 }
5658 Ok(o) => Some(format!(
5659 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5660 someone may have pushed): {}",
5661 short(&theirs),
5662 o.stderr.chars().take(600).collect::<String>()
5663 )),
5664 Err(e) => Some(format!(
5665 "{branch} is rebased locally but could not be pushed: {e:#}"
5666 )),
5667 }
5668 }
5669
5670 async fn settle_already_in(
5687 &mut self,
5688 branch: &str,
5689 tip: &str,
5690 head: &str,
5691 attempts: usize,
5692 behind: usize,
5693 ) -> Result<bool> {
5694 let repo = self.state.repo.clone();
5695 let remote = self.state.config.merge.remote.clone();
5696 let start = self.state.base_commit.clone();
5697 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5698 Ok(Some(e)) => e,
5699 Ok(None) => return Ok(false),
5700 Err(e) => {
5701 tracing::warn!("already-in-base check for {branch}: {e:#}");
5702 return Ok(false);
5703 }
5704 };
5705 let mut verified = vec![head.to_owned()];
5706 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5707 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5708 if matches!(&fetched, Ok(o) if o.ok())
5709 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5710 && theirs != head
5711 {
5712 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5713 Ok(Some(_)) => verified.push(theirs),
5714 _ => return Ok(false),
5715 }
5716 }
5717 let base_branch = self.state.base_branch.clone();
5718 let message = format!(
5719 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5720 land",
5721 evidence.names(),
5722 evidence.proof.as_str()
5723 );
5724 let closed =
5725 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5726 .await;
5727 match closed {
5728 Ok(Ok(url)) => self
5729 .state
5730 .event("land", format!("closed {url}: superseded on {base_branch}")),
5731 Ok(Err(why)) => self
5732 .state
5733 .event("land", format!("did not close a pull request: {why}")),
5734 Err(e) => {
5735 let why = format!(
5736 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5737 not be closed ({e:#}); resume to retry"
5738 );
5739 self.state.status = RunStatus::Blocked;
5740 self.state.base_sync = Some(BaseSync {
5741 tip: tip.to_owned(),
5742 behind,
5743 attempts,
5744 conflict: Some(why.clone()),
5745 already_in: None,
5746 });
5747 self.state.event("land", why);
5748 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5749 .await?;
5750 self.state.save()?;
5751 return Ok(true);
5752 }
5753 }
5754 self.state.status = RunStatus::AlreadyInBase;
5755 self.state.base_sync_recovery.stop = None;
5756 self.state.base_sync = Some(BaseSync {
5757 tip: tip.to_owned(),
5758 behind,
5759 attempts,
5760 conflict: None,
5761 already_in: Some(evidence),
5762 });
5763 self.state.event("land", message);
5764 self.state.save()?;
5765 self.settle_questions();
5766 Ok(true)
5767 }
5768
5769 fn landing_base(&self) -> String {
5779 self.state
5780 .base_sync
5781 .as_ref()
5782 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5783 }
5784
5785 pub async fn fix_selected(
5818 &mut self,
5819 ids: &[String],
5820 reason: &str,
5821 allow_stale: bool,
5822 ) -> Result<()> {
5823 let reason = reason.trim();
5824 if reason.is_empty() {
5825 bail!("a fix request needs a reason — that is the operator's own record of why");
5826 }
5827 if ids.is_empty() {
5828 bail!("no finding id given");
5829 }
5830 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5831 bail!(
5832 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5833 has already concluded — can be given a targeted fix. A run still \
5834 in progress should simply be resumed; a `merged` run's branch has \
5835 already landed, so its answer is a fresh `magi review <branch>`, \
5836 not reopening this run's own record",
5837 self.state.id,
5838 self.state.status.as_str()
5839 );
5840 }
5841 let Some(winner) = self.state.winner().cloned() else {
5842 bail!("run {} has no winning candidate to fix", self.state.id);
5843 };
5844 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5845 bail!(
5846 "branch `{}` no longer exists; this run cannot be extended",
5847 winner.branch
5848 );
5849 }
5850 let home = crate::run::home();
5851 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5852 bail!(
5853 "run {} is currently being worked on by another magi process",
5854 self.state.id
5855 );
5856 }
5857 let _claim = FixClaim::acquire(&self.state.dir())?;
5863
5864 let mut seen = BTreeSet::new();
5868 let mut findings = Vec::new();
5869 let mut missing = Vec::new();
5870 for id in ids {
5871 if !seen.insert(id.clone()) {
5872 continue;
5873 }
5874 match self.state.finding(id) {
5875 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5876 id: f.id.clone(),
5877 severity: f.severity,
5878 reviewer_vote: rec.vote,
5879 round: round.round,
5880 round_head: round.head.clone(),
5881 reviewer: rec.reviewer,
5882 agent: rec.agent.clone(),
5883 file: f.file.clone(),
5884 line: f.line,
5885 title: f.title.clone(),
5886 detail: f.detail.clone(),
5887 outcome: OperatorFixOutcome::Pending,
5888 }),
5889 None => missing.push(id.clone()),
5890 }
5891 }
5892 if !missing.is_empty() {
5893 bail!(
5894 "unknown finding id(s): {}; nothing was changed",
5895 missing.join(", ")
5896 );
5897 }
5898
5899 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5900 let stale_details: Vec<(String, String)> = findings
5901 .iter()
5902 .filter(|f| f.round_head != head_at_request)
5903 .map(|f| (f.id.clone(), f.round_head.clone()))
5904 .collect();
5905 let stale = !stale_details.is_empty();
5906 if stale && !allow_stale {
5907 bail!(
5908 "the branch has moved since some finding(s) were raised — {} — now \
5909 at {}; pass --allow-stale to fix anyway, or re-run review first",
5910 stale_details
5911 .iter()
5912 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5913 .collect::<Vec<_>>()
5914 .join(", "),
5915 short(&head_at_request)
5916 );
5917 }
5918
5919 let request = OperatorFixRequest {
5920 requested_at: Timestamp::now(),
5921 reason: reason.to_owned(),
5922 findings,
5923 head_at_request: head_at_request.clone(),
5924 allow_stale,
5925 stale,
5926 fix: None,
5927 result_head: None,
5928 follow_up_review_run: None,
5929 };
5930 self.state.event(
5931 "fix",
5932 format!(
5933 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5934 request.findings.len(),
5935 request
5936 .findings
5937 .iter()
5938 .map(|f| f.id.as_str())
5939 .collect::<Vec<_>>()
5940 .join(", "),
5941 ),
5942 );
5943 self.state.operator_fixes.push(request);
5950 self.state.save()?;
5951 let request_index = self.state.operator_fixes.len() - 1;
5952
5953 let repo = self.state.repo.clone();
5962 let coordination = self.coordinate(&repo).await?;
5963 if winner.worktree.exists() {
5964 let dirty = git::git(
5967 &winner.worktree,
5968 &["status", "--porcelain", "--untracked-files=all"],
5969 )
5970 .await?;
5971 let only_withheld = dirty.lines().all(|l| {
5972 l.strip_prefix("?? ")
5973 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5974 });
5975 if !only_withheld {
5976 bail!(
5977 "`{}` has uncommitted changes; refusing to touch it — commit or \
5978 discard them first",
5979 winner.worktree.display()
5980 );
5981 }
5982 git::worktree_remove(&self.state.repo, &winner.worktree)
5983 .await
5984 .ok();
5985 }
5986 let fix_worktree = self.state.worktree_root().join("operator-fix");
5987 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5988 git::git(
5989 &self.state.repo,
5990 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5991 )
5992 .await
5993 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5994 if !git::is_clean(&fix_worktree).await? {
5995 git::worktree_remove(&self.state.repo, &fix_worktree)
5996 .await
5997 .ok();
5998 bail!(
5999 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6000 winner.branch
6001 );
6002 }
6003 let repo_for_setup = self.state.repo.clone();
6004 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6005 Ok(()) => {
6006 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6007 }
6008 Err(e) => Err(e),
6009 };
6010 if let Err(e) = setup {
6011 git::worktree_remove(&self.state.repo, &fix_worktree)
6012 .await
6013 .ok();
6014 return Err(e);
6015 }
6016
6017 self.state.resource_block = None;
6020
6021 let run_id = self.state.id.clone();
6022 let prompts = self.state.config.prompts.clone();
6023 let language = self.state.config.graph.language.clone();
6024 let sessions = self.state.config.graph.sessions;
6025 let artifacts = agent::artifacts_dir(&self.state.dir());
6026 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6027 .findings
6028 .iter()
6029 .map(|f| Finding {
6030 id: f.id.clone(),
6031 severity: f.severity,
6032 file: f.file.clone(),
6033 line: f.line,
6034 title: f.title.clone(),
6035 detail: f.detail.clone(),
6036 })
6037 .collect();
6038 let fix_prompt = prompt::operator_fix_in(
6039 &self.state.instruction,
6040 &finding_list,
6041 reason,
6042 &stale_details,
6043 &head_at_request,
6044 &language,
6045 &self.state.config.graph.github_language,
6046 );
6047 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6048 drop(coordination);
6049 let (job, seat, out) = self
6050 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6051 prompt: fix_prompt.clone(),
6052 spec,
6053 seat,
6054 cwd: fix_worktree.clone(),
6055 timeout,
6056 allow_write: true,
6057 sessions,
6058 artifacts: artifacts.clone(),
6059 stem: "operator-fix".to_owned(),
6060 handover: None,
6061 })
6062 .await;
6063 let agent_id = seat.agent.clone();
6064
6065 let mut fix = FixRecord {
6066 agent: agent_id,
6067 addressed: Vec::new(),
6068 rejected: Vec::new(),
6069 notes: String::new(),
6070 committed: false,
6071 failed: None,
6072 duration_ms: 0,
6073 continuation: None,
6074 };
6075 let mut final_seat = seat.clone();
6076 match out {
6077 AgentOutcome::Ok(o) => {
6078 fix.duration_ms = o.duration_ms;
6079 let parsed = verdict::extract_json::<FixReport>(&o.text);
6080 let incomplete_reason = match &parsed {
6081 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6082 "the reply parsed, but it reported a command whose own CLI \
6083 never confirmed an exit status"
6084 .to_owned(),
6085 ),
6086 Ok(_) => None,
6087 Err(e) => Some(e.to_string()),
6088 };
6089 match incomplete_reason {
6090 None => {
6091 let report = parsed.expect("checked Ok above");
6092 fix.addressed = report.addressed;
6093 fix.rejected = report.rejected;
6094 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6095 }
6096 Some(reason) => {
6097 let (resumed_seat, resolved, failure, cont) = self
6098 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6099 .await;
6100 fix.duration_ms += cont.cumulative_wait_ms;
6101 fix.continuation = Some(cont);
6102 final_seat = resumed_seat;
6103 match resolved {
6104 Some(report) => {
6105 fix.addressed = report.addressed;
6106 fix.rejected = report.rejected;
6107 fix.notes =
6108 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6109 }
6110 None => fix.failed = failure,
6111 }
6112 }
6113 }
6114 }
6115 AgentOutcome::Dropped(o) => {
6116 fix.duration_ms = o.duration_ms;
6117 let why = o
6118 .dropped
6119 .as_ref()
6120 .map(|d| d.why.as_str())
6121 .unwrap_or("the CLI ended the stream without delivering its answer");
6122 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6123 }
6124 AgentOutcome::Quota(o) => {
6125 self.state.quota.push(QuotaLoss::from_report(
6126 final_seat.key.clone(),
6127 "fix",
6128 o.quota.as_ref(),
6129 ));
6130 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6131 }
6132 AgentOutcome::Failed(e) => fix.failed = Some(e),
6133 AgentOutcome::ResourceBlocked(e) => {
6134 fix.failed = Some(format!("the fixer was not started: {e}"));
6135 }
6136 }
6137 if fix.continuation.is_none() {
6140 fix.continuation = Some(ContinuationRecord::not_needed());
6141 }
6142 self.state.seats.insert(final_seat.key.clone(), final_seat);
6143
6144 let rescue_message = format!(
6145 "magi: operator-selected fix ({}) (uncommitted work)",
6146 self.state.operator_fixes[request_index]
6147 .findings
6148 .iter()
6149 .map(|f| f.id.as_str())
6150 .collect::<Vec<_>>()
6151 .join(", ")
6152 );
6153 let repo = self.state.repo.clone();
6154 let coordination = self.coordinate(&repo).await?;
6155 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6156 self.state.note_withheld("fix", &r.withheld);
6157 }
6158 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6159 fix.committed = after != head_at_request;
6160 git::worktree_remove(&self.state.repo, &fix_worktree)
6161 .await
6162 .ok();
6163 drop(coordination);
6164
6165 self.state.event(
6166 "fix",
6167 match &fix.failed {
6168 Some(reason) => format!(
6169 "operator fix: adoption report was lost ({reason}); {}",
6170 if fix.committed {
6171 "committed"
6172 } else {
6173 "NO new commit"
6174 }
6175 ),
6176 None => format!(
6177 "operator fix: {} addressed, {} rejected, {}",
6178 fix.addressed.len(),
6179 fix.rejected.len(),
6180 if fix.committed {
6181 "committed"
6182 } else {
6183 "NO new commit"
6184 }
6185 ),
6186 },
6187 );
6188
6189 for f in &mut self.state.operator_fixes[request_index].findings {
6196 f.outcome = if fix.failed.is_some() {
6197 OperatorFixOutcome::Unreported
6198 } else if fix.addressed.contains(&f.id) {
6199 OperatorFixOutcome::Addressed
6200 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6201 OperatorFixOutcome::Rejected { why: r.why.clone() }
6202 } else {
6203 OperatorFixOutcome::Unreported
6204 };
6205 }
6206
6207 let committed = fix.committed;
6208 if committed {
6209 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6210 }
6211 self.state.operator_fixes[request_index].fix = Some(fix);
6212 self.state.save()?;
6215 if self.state.resource_block.is_some() {
6216 return self.stop_for_resource();
6217 }
6218
6219 if committed {
6220 self.state.event(
6221 "fix",
6222 format!(
6223 "operator fix committed {}; opening a follow-up review-only run",
6224 short(&after)
6225 ),
6226 );
6227 let origin =
6230 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6231 match Self::review(
6232 &self.state.repo,
6233 &winner.branch,
6234 self.state.config.clone(),
6235 origin,
6236 )
6237 .await
6238 {
6239 Ok(mut follow_up) => {
6240 follow_up.state.event(
6241 "start",
6242 format!(
6243 "requested by an operator fix on run {} for finding(s) {}",
6244 self.state.id,
6245 self.state.operator_fixes[request_index]
6246 .findings
6247 .iter()
6248 .map(|f| f.id.as_str())
6249 .collect::<Vec<_>>()
6250 .join(", "),
6251 ),
6252 );
6253 follow_up.state.save()?;
6254 let follow_up_id = follow_up.state.id.clone();
6255 let adopted = match crate::direct::adopt(
6258 &follow_up.state,
6259 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6260 ) {
6261 Ok(a) => a,
6262 Err(e) => {
6263 self.state.event(
6266 "fix",
6267 format!(
6268 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6269 ),
6270 );
6271 self.state.save()?;
6272 return Ok(());
6273 }
6274 };
6275 let executed = follow_up.execute().await;
6276 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6277 if let Some(a) = adopted {
6278 a.finish(&follow_up.state, executed);
6279 }
6280 if let Some(e) = failure {
6281 self.state.event(
6282 "fix",
6283 format!(
6284 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6285 ),
6286 );
6287 }
6288 self.state.operator_fixes[request_index].follow_up_review_run =
6289 Some(follow_up_id);
6290 }
6291 Err(e) => {
6292 self.state.event(
6293 "fix",
6294 format!("committed the fix but could not open a follow-up review: {e:#}"),
6295 );
6296 }
6297 }
6298 self.state.save()?;
6299 }
6300
6301 Ok(())
6302 }
6303
6304 async fn ask_fixer(
6324 &mut self,
6325 winner: &Candidate,
6326 node: &'static str,
6327 round: Option<usize>,
6328 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6329 ) -> (SeatJob, SeatState, AgentOutcome) {
6330 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6331 self.ask_fixer_among(attempts, node, round, None, build)
6332 .await
6333 }
6334
6335 async fn ask_fixer_among(
6340 &mut self,
6341 attempts: Vec<(AgentSpec, String)>,
6342 node: &'static str,
6343 round: Option<usize>,
6344 restore: Option<(&Path, &str)>,
6345 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6346 ) -> (SeatJob, SeatState, AgentOutcome) {
6347 let run_id = self.state.id.clone();
6348 let prompts = self.state.config.prompts.clone();
6349 let cache = self.state.config.cache_dir();
6350 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6351 let mut last = None;
6352 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6353 let seat = self.seat(&key, &spec.id);
6354 let mut job = build(spec.clone(), seat);
6355 if i > 0 {
6356 job.stem = format!("{}-{}", job.stem, spec.id);
6357 }
6358 let ctx = WaveCtx {
6359 carry_seats: false,
6360 pauses: Some((&self.pause, &self.interrupt)),
6361 run: &run_id,
6362 node,
6363 prompts: &prompts,
6364 cache: cache.as_deref(),
6365 round,
6366 };
6367 let (seat, out) =
6368 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6369 let advances = match &out {
6372 AgentOutcome::Ok(o) => agent::output_advances(o),
6373 AgentOutcome::ResourceBlocked(_) => false,
6374 _ => true,
6375 };
6376 if let Some(next) = ids.get(i + 1)
6377 && advances
6378 {
6379 self.state.seats.insert(seat.key.clone(), seat.clone());
6380 let class =
6381 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6382 let reason = match &out {
6383 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6384 other => fail_reason(other),
6385 };
6386 if let Some((wt, rev)) = restore
6390 && let Err(e) = self.graft_restore(wt, rev).await
6391 {
6392 let why =
6393 format!("could not restore the worktree before the next fixer: {e:#}");
6394 tracing::warn!("{why}");
6395 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6396 AgentOutcome::ResourceBlocked(why)
6397 } else {
6398 AgentOutcome::Failed(why)
6399 };
6400 last = Some((job, seat, out));
6401 break;
6402 }
6403 record_handover(
6404 &mut self.state,
6405 node,
6406 &seat.key,
6407 &spec.id,
6408 next,
6409 &class,
6410 &reason,
6411 );
6412 continue;
6413 }
6414 last = Some((job, seat, out));
6415 break;
6416 }
6417 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6418 if let AgentOutcome::Quota(output) = &out
6419 && node != "graft"
6420 {
6421 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6426 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6427 self.state.quota.push(loss);
6428 self.state.seats.insert(seat.key.clone(), seat.clone());
6429 self.state.status = RunStatus::Blocked;
6430 return (
6431 job,
6432 seat,
6433 AgentOutcome::ResourceBlocked(
6434 "fixer unavailable; restore agent availability and resume".into(),
6435 ),
6436 );
6437 }
6438 (job, seat, out)
6439 }
6440
6441 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6452 use crate::recovery::{Mode, Repair};
6453 if !self.state.reviews.is_empty() {
6454 return Ok(());
6455 }
6456 let Some(mut rec) = self.state.recovery.clone() else {
6457 return Ok(());
6458 };
6459 if rec.mode != Mode::VerifyRepair {
6460 return Ok(());
6461 }
6462 let wt = winner.worktree.clone();
6463 let mut head = git::rev_parse(&wt, "HEAD").await?;
6464 match rec.repair.clone() {
6465 Repair::Done { .. } => return Ok(()),
6466 Repair::WaitingForRestore { before } => {
6467 self.graft_restore(&wt, &before).await?;
6468 head = before;
6469 rec.repair = Repair::NotStarted;
6470 self.state.recovery = Some(rec.clone());
6471 self.state.save()?;
6472 }
6473 Repair::Started { before } => {
6474 if let Err(e) = self.graft_restore(&wt, &before).await {
6478 if e.is::<crate::repo_lock::ResourceBlocked>() {
6479 return Err(e);
6480 }
6481 return self
6482 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6483 .await;
6484 }
6485 rec.repair = Repair::Done {
6486 before: before.clone(),
6487 after: before,
6488 committed: false,
6489 failed: Some("interrupted; rolled back".to_owned()),
6490 };
6491 self.state
6492 .event("recovery", "the repair was interrupted and rolled back");
6493 self.state.recovery = Some(rec);
6494 self.state.save()?;
6495 return Ok(());
6496 }
6497 Repair::NotStarted => {}
6498 }
6499 if crate::recovery::recheck(&mut rec, &head) {
6500 self.state.event("recovery", rec.summary());
6501 self.state.recovery = Some(rec);
6502 self.state.save()?;
6503 return Ok(());
6504 }
6505 rec.repair = Repair::Started {
6506 before: head.clone(),
6507 };
6508 self.state.recovery = Some(rec.clone());
6509 self.state.save()?;
6510
6511 let language = self.state.config.graph.language.clone();
6512 let sessions = self.state.config.graph.sessions;
6513 let artifacts = agent::artifacts_dir(&self.state.dir());
6514 let fix_prompt = prompt::recovery_fix(
6515 &self.state.instruction,
6516 &rec,
6517 self.state.config.graph.review_rounds,
6518 &language,
6519 );
6520 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6521 self.state.event(
6522 "recovery",
6523 format!(
6524 "repair: the fixer starts from the failed verification on {}",
6525 short(&head)
6526 ),
6527 );
6528 let (_job, seat, out) = self
6529 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6530 prompt: fix_prompt.clone(),
6531 spec,
6532 seat,
6533 cwd: wt.clone(),
6534 timeout,
6535 allow_write: true,
6536 sessions,
6537 artifacts: artifacts.clone(),
6538 stem: "fix-recovery".to_owned(),
6539 handover: None,
6540 })
6541 .await;
6542 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6543 rec.repair = Repair::WaitingForRestore { before: head };
6544 self.state.recovery = Some(rec);
6545 return self.stop_for_resource();
6546 }
6547 let failed = match &out {
6548 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6549 AgentOutcome::Quota(o) => {
6550 self.state.quota.push(QuotaLoss::from_report(
6551 seat.key.clone(),
6552 "fix",
6553 o.quota.as_ref(),
6554 ));
6555 Some("rate limited (quota); the fixer could not run".to_owned())
6556 }
6557 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6558 AgentOutcome::Failed(e) => Some(e.clone()),
6559 };
6560 self.state.seats.insert(seat.key.clone(), seat);
6561 let coordination = self.coordinate(&wt).await?;
6562 let failed = match failed {
6563 Some(why) => {
6565 if let Err(e) = git::restore_to(&wt, &head).await {
6566 return self
6567 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6568 .await;
6569 }
6570 Some(why)
6571 }
6572 None => {
6573 if let Ok(r) = git::rescue_commit(
6574 &wt,
6575 "magi: recovery repair of the failed verification (uncommitted work)",
6576 )
6577 .await
6578 {
6579 self.state.note_withheld("fix", &r.withheld);
6580 }
6581 None
6582 }
6583 };
6584 let after = git::rev_parse(&wt, "HEAD").await?;
6585 drop(coordination);
6586 self.state.event(
6587 "recovery",
6588 match &failed {
6589 Some(why) => format!("repair: nothing kept ({why})"),
6590 None if after != head => {
6591 format!("repair: committed {}", short(&after))
6592 }
6593 None => "repair: the fixer made no commit".to_owned(),
6594 },
6595 );
6596 rec.repair = Repair::Done {
6597 before: head.clone(),
6598 committed: after != head,
6599 after,
6600 failed,
6601 };
6602 self.state.recovery = Some(rec);
6603 self.state.save()?;
6604 Ok(())
6605 }
6606
6607 async fn stop_recovery(
6610 &mut self,
6611 mut rec: crate::recovery::RecoveryRecord,
6612 head: &str,
6613 why: String,
6614 ) -> Result<()> {
6615 rec.repair = crate::recovery::Repair::Started {
6617 before: head.to_owned(),
6618 };
6619 self.state.recovery = Some(rec);
6620 self.state
6621 .event("recovery", format!("repair could not be undone: {why}"));
6622 self.state.status = RunStatus::Blocked;
6623 self.state.save()?;
6624 bail!("recovery repair could not be rolled back: {why}")
6625 }
6626
6627 async fn review_loop(&mut self) -> Result<()> {
6628 if self.hold_base_sync().await? {
6629 return Ok(());
6630 }
6631 loop {
6632 if self
6633 .state
6634 .base_sync
6635 .as_ref()
6636 .is_some_and(|s| s.conflict.is_some())
6637 {
6638 return Ok(());
6639 }
6640 if self.catchup_repairable() {
6641 let record = self.state.reviews.last().expect("repairable round").clone();
6642 let winner = self
6643 .state
6644 .winner()
6645 .cloned()
6646 .context("catch-up repair has no winner")?;
6647 if !self.fix_review_round(&winner, record).await? {
6648 return Ok(());
6649 }
6650 }
6651 self.review_loop_pass().await?;
6652 if !self.catchup_repairable() {
6653 return Ok(());
6654 }
6655 }
6656 }
6657
6658 fn catchup_repairable(&self) -> bool {
6661 self.state.reviews.len() < self.state.config.graph.review_rounds
6662 && self.state.reviews.last().is_some_and(|r| {
6663 !r.clean
6664 && r.blocking == 0
6665 && r.fix.is_none()
6666 && r.e2e_status() == E2eStatus::Failed
6667 })
6668 }
6669
6670 async fn review_loop_pass(&mut self) -> Result<()> {
6671 if self
6676 .state
6677 .base_sync
6678 .as_ref()
6679 .is_some_and(|s| s.conflict.is_some())
6680 {
6681 return Ok(());
6682 }
6683 let run_id = self.state.id.clone();
6688 let prompts = self.state.config.prompts.clone();
6689 let Some(winner) = self.state.winner().cloned() else {
6690 return Ok(());
6691 };
6692 let granted_review = self
6693 .state
6694 .repair_grants
6695 .last()
6696 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6697 .cloned();
6698 let fresh_review = granted_review
6699 .as_ref()
6700 .is_some_and(|g| self.state.reviews.len() == g.review_start);
6701 let max_rounds = granted_review
6702 .as_ref()
6703 .map_or(self.state.config.graph.review_rounds, |g| {
6704 g.review_start.saturating_add(1)
6705 });
6706 if let Some(status) =
6716 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
6717 {
6718 if status == RunStatus::Gating {
6721 self.record_contested_handoff();
6722 }
6723 self.state.status = status;
6724 self.state.save()?;
6725 return Ok(());
6726 }
6727 self.state.status = RunStatus::Reviewing;
6728 if !fresh_review
6738 && self.state.reviews.last().is_some_and(|r| {
6739 r.e2e_status() == E2eStatus::ResourceBlocked
6740 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6741 })
6742 {
6743 let shell = self.state.config.shell();
6744 return self
6745 .stop_unresolved_review(
6746 "the last round's own verification never resolved",
6747 &shell,
6748 &winner.worktree,
6749 )
6750 .await;
6751 }
6752
6753 if granted_review.is_none() {
6754 self.recovery_repair(&winner).await?;
6755 }
6756 if self.resource_halt()? {
6757 return Ok(());
6758 }
6759
6760 let repo = self.state.repo.clone();
6761 let root = self.state.worktree_root();
6762 let language = self.state.config.graph.language.clone();
6763 let sessions = self.state.config.graph.sessions;
6764 let artifacts = self.merge_repair_artifacts();
6765 let base = self.landing_base();
6766 let base_short = short(&base);
6767 let reviewers = self.roles.reviewers.clone();
6768 let shell = self.state.config.shell();
6769
6770 for round in (self.state.reviews.len() + 1)..=max_rounds {
6771 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6772 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6773 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6774 let prev_verification = self
6783 .state
6784 .reviews
6785 .last()
6786 .and_then(|r| r.verification_summary(&head));
6787 let prior_section = match (&self.state.recovery, round) {
6790 (Some(rec), 1) if self.state.reviews.is_empty() => {
6791 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6792 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6793 Some(crate::recovery::head_of(
6794 &git::diff(&winner.worktree, reviewed, "HEAD")
6795 .await
6796 .unwrap_or_default(),
6797 crate::recovery::SINCE_DIFF_MAX,
6798 ))
6799 }
6800 _ => None,
6801 };
6802 Some(prompt::recovery_review(rec, since.as_deref()))
6803 }
6804 _ => None,
6805 };
6806
6807 let mut jobs = Vec::new();
6811 let coordination = self.coordinate(&repo).await?;
6812 for (r, spec) in reviewers.iter().cloned().enumerate() {
6813 let wt = root.join(format!("review-{}", r + 1));
6814 if wt.exists() {
6817 git::reset_detached(&wt, &head).await?;
6818 } else {
6819 git::worktree_add_detached(&repo, &wt, &head).await?;
6820 }
6821 self.setup_seat_worktree(&repo, &wt, false).await?;
6822 let seat_key = format!("review-{}", r + 1);
6823 let spec = pick_start_spec(
6826 &self.roles.reviewer_roster,
6827 spec,
6828 self.state.seat_history.get(&seat_key),
6829 );
6830 let seat = self.seat(&seat_key, &spec.id);
6831 jobs.push(SeatJob {
6832 prompt: prompt::review(&prompt::ReviewCtx {
6833 instruction: &self.state.instruction,
6834 branch: &winner.branch,
6835 base_short: &base_short,
6836 stat: &stat,
6837 patch: &patch,
6838 verification: prev_verification.as_ref(),
6839 prior: prior_section.as_deref(),
6840 reviewers: reviewers.len(),
6841 round,
6842 rounds: max_rounds,
6843 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6846 lens: Lens::for_seat(r),
6847 language: &language,
6848 github_language: &self.state.config.graph.github_language,
6849 }),
6850 spec,
6851 seat,
6852 cwd: wt,
6853 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6854 allow_write: false,
6855 sessions,
6856 artifacts: artifacts.clone(),
6857 stem: format!("review-{round}-{}", r + 1),
6858 handover: None,
6859 });
6860 }
6861
6862 drop(coordination);
6863 self.state.event(
6864 "review",
6865 format!(
6866 "round {round}: {} reviewers on {}",
6867 jobs.len(),
6868 short(&head)
6869 ),
6870 );
6871 let mut quota_losses = Vec::new();
6872 let review_retries = self.state.config.graph.retries;
6873 let review_cache = self.state.config.cache_dir();
6874 let ctx = WaveCtx {
6875 carry_seats: true,
6876 pauses: Some((&self.pause, &self.interrupt)),
6877 run: &run_id,
6878 node: "review",
6879 prompts: &prompts,
6880 cache: review_cache.as_deref(),
6881 round: Some(round),
6882 };
6883 let results = ask_json_wave::<Review>(
6884 jobs,
6885 Arc::clone(&self.sem),
6886 review_retries,
6887 &self.roles.reviewer_roster,
6888 &ctx,
6889 &mut quota_losses,
6890 &mut self.state,
6891 &|_: &Review| Ok(()),
6892 )
6893 .await;
6894 if self.state.resource_block.is_some() {
6895 return self.stop_for_resource();
6896 }
6897 let round_quota_missing = quota_losses.len();
6901 self.state.quota.extend(quota_losses);
6902
6903 let mut records = Vec::new();
6904 let mut all_findings = Vec::new();
6905 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6906 let agent_id = seat.agent.clone();
6907 self.state.seats.insert(seat.key.clone(), seat);
6908 let mut record = ReviewRecord {
6909 reviewer: r + 1,
6910 agent: agent_id,
6911 summary: String::new(),
6912 findings: Vec::new(),
6913 vote: None,
6914 failed: None,
6915 duration_ms: 0,
6916 attempts,
6922 };
6923 match res {
6924 Ok((review, out)) => {
6925 record.summary =
6933 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6934 record.vote = Some(review.vote);
6935 record.duration_ms = out.duration_ms;
6936 for (n, mut f) in review.findings.into_iter().enumerate() {
6937 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6940 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6941 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6942 f.file = f
6948 .file
6949 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6950 all_findings.push(f.clone());
6951 record.findings.push(f);
6952 }
6953 self.state.event(
6954 "review",
6955 format!(
6956 "round {round}: reviewer {} voted {} with {} finding(s)",
6957 r + 1,
6958 review.vote.label(),
6959 record.findings.len()
6960 ),
6961 );
6962 }
6963 Err(e) => {
6964 record.failed = Some(e.to_string());
6965 self.state.event(
6966 "review",
6967 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6968 );
6969 }
6970 }
6971 records.push(record);
6972 }
6973
6974 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6981 let vote_split =
6982 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6983 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6984 if vote_split {
6985 self.state.event(
6986 "review",
6987 format!(
6988 "round {round}: votes split ({}) — one round of reconsideration",
6989 initial_votes
6990 .iter()
6991 .map(|v| v.label())
6992 .collect::<Vec<_>>()
6993 .join(", ")
6994 ),
6995 );
6996 let panel: Vec<ReviewSeatReport<'_>> = records
6999 .iter()
7000 .filter_map(|r| {
7001 r.vote.map(|vote| ReviewSeatReport {
7002 reviewer: r.reviewer,
7003 vote,
7004 summary: &r.summary,
7005 findings: &r.findings,
7006 })
7007 })
7008 .collect();
7009
7010 let mut jobs = Vec::new();
7011 let mut seats_at = Vec::new();
7012 for (r, spec) in reviewers.iter().cloned().enumerate() {
7013 if records[r].vote.is_none() {
7017 continue;
7018 }
7019 let wt = root.join(format!("review-{}", r + 1));
7020 let seat_key = format!("review-{}", r + 1);
7021 let spec = self.occupant(&seat_key, spec);
7022 let seat = self.seat(&seat_key, &spec.id);
7023 let build = |with_patch: bool| {
7031 prompt::review_reconsider(&ReviewReconsiderCtx {
7032 instruction: &self.state.instruction,
7033 reviewer: r + 1,
7034 lens: Lens::for_seat(r),
7035 panel: &panel,
7036 patch: with_patch.then_some(ReviewPatch {
7037 branch: &winner.branch,
7038 base_short: &base_short,
7039 stat: &stat,
7040 patch: &patch,
7041 }),
7042 round,
7043 rounds: max_rounds,
7044 language: &language,
7045 })
7046 };
7047 let full = build(true);
7048 let prompt = if has_context(&spec, &seat, sessions) {
7049 build(false)
7050 } else {
7051 full.clone()
7052 };
7053 jobs.push(SeatJob {
7054 prompt,
7055 spec,
7056 seat,
7057 cwd: wt,
7058 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7059 allow_write: false,
7060 sessions,
7061 artifacts: artifacts.clone(),
7062 stem: format!("review-{round}-reconsider-{}", r + 1),
7063 handover: Some(full),
7064 });
7065 seats_at.push(r);
7066 }
7067
7068 let mut recon_quota_losses = Vec::new();
7069 let recon_cache = self.state.config.cache_dir();
7070 let recon_ctx = WaveCtx {
7071 carry_seats: true,
7072 pauses: Some((&self.pause, &self.interrupt)),
7073 run: &run_id,
7074 node: "review",
7075 prompts: &prompts,
7076 cache: recon_cache.as_deref(),
7077 round: Some(round),
7078 };
7079 let recon_results = ask_json_wave::<ReviewRevote>(
7080 jobs,
7081 Arc::clone(&self.sem),
7082 review_retries,
7083 &self.roles.reviewer_roster,
7084 &recon_ctx,
7085 &mut recon_quota_losses,
7086 &mut self.state,
7087 &|_: &ReviewRevote| Ok(()),
7088 )
7089 .await;
7090 if self.state.resource_block.is_some() {
7091 return self.stop_for_resource();
7092 }
7093 self.state.quota.extend(recon_quota_losses);
7094
7095 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7096 let agent_id = seat.agent.clone();
7097 self.state.seats.insert(seat.key.clone(), seat);
7098 let mut rec = ReviewRevoteRecord {
7099 reviewer: r + 1,
7100 agent: agent_id,
7101 vote: None,
7102 reason: String::new(),
7103 failed: None,
7104 };
7105 match res {
7106 Ok((rv, _)) => {
7107 rec.vote = Some(rv.vote);
7108 rec.reason =
7109 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7110 self.state.event(
7111 "review",
7112 format!(
7113 "round {round}: reviewer {} revoted {}",
7114 r + 1,
7115 rv.vote.label()
7116 ),
7117 );
7118 }
7119 Err(e) => {
7120 rec.failed = Some(e.to_string());
7121 self.state.event(
7122 "review",
7123 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7124 );
7125 }
7126 }
7127 reconsideration.push(rec);
7128 }
7129 } else if initial_votes.len() > 1 {
7130 self.state.event(
7131 "review",
7132 format!(
7133 "round {round}: votes agreed ({}) — no reconsideration",
7134 initial_votes[0].label()
7135 ),
7136 );
7137 }
7138
7139 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7140 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7141 let defer_e2e =
7152 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7153 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7154 let reason =
7155 format!("{blocking} blocking finding(s) already required a fix this round");
7156 self.state.event(
7157 "verify",
7158 format!(
7159 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7160 {}); it will run once a round has none left",
7161 short(&head)
7162 ),
7163 );
7164 (Vec::new(), false, true, Some(reason))
7165 } else {
7166 let e2e_commands = self.state.config.verify.e2e.clone();
7167 let cache_dir = self.state.config.cache_dir();
7168 let context = format!("round {round}");
7169 let wait = self.repository_wait();
7170 let (e2e, verify_retried) = with_cache_lease(
7171 &mut self.state,
7172 cache_dir.as_deref(),
7173 "e2e",
7174 "e2e",
7175 &winner.worktree,
7176 &head,
7177 verify_timeout,
7178 &context,
7179 &wait,
7180 |state, budget| {
7181 let shell = shell.clone();
7182 let e2e_commands = e2e_commands.clone();
7183 let worktree = winner.worktree.clone();
7184 let context = context.clone();
7185 async move {
7186 run_e2e_with_retry(
7187 state,
7188 &shell,
7189 &e2e_commands,
7190 &worktree,
7191 budget,
7192 &context,
7193 )
7194 .await
7195 }
7196 },
7197 )
7198 .await?;
7199 (e2e, verify_retried, false, None)
7200 };
7201
7202 let expected = records.len();
7203 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7204 let incomplete = answered < expected;
7205 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7206 let policy = self.state.config.graph.incomplete_review;
7207 let clean = round_is_clean(
7208 blocking,
7209 e2e_ok,
7210 answered,
7211 expected,
7212 round_quota_missing,
7213 policy,
7214 );
7215
7216 let mut round_record = ReviewRound {
7217 round,
7218 head: head.clone(),
7219 verified_head: None,
7220 verified_at: None,
7221 reviews: records,
7222 e2e,
7223 verify_retried,
7224 e2e_deferred,
7225 e2e_defer_reason,
7226 fix: None,
7227 blocking,
7228 answered,
7229 expected,
7230 clean,
7231 progressed: false,
7232 vote_split,
7233 reconsideration,
7234 verdict: None,
7235 };
7236 round_record.verdict = ReviewVote::worst(
7240 round_record
7241 .final_votes()
7242 .into_iter()
7243 .map(|(_, _, vote)| vote),
7244 );
7245 if !matches!(
7256 round_record.e2e_status(),
7257 E2eStatus::Deferred | E2eStatus::NotConfigured
7258 ) {
7259 round_record.verified_head = Some(head.clone());
7260 round_record.verified_at = Some(Timestamp::now());
7261 }
7262
7263 if incomplete {
7264 let missing: Vec<String> = round_record
7265 .reviews
7266 .iter()
7267 .filter(|r| r.failed.is_some())
7268 .map(|r| format!("review-{}", r.reviewer))
7269 .collect();
7270 self.state.event(
7271 "review",
7272 format!(
7273 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7274 missing.join(", ")
7275 ),
7276 );
7277 }
7278
7279 if clean {
7280 self.state.event(
7281 "review",
7282 if incomplete && policy == IncompleteReviewPolicy::Warn {
7283 format!(
7284 "round {round}: clean (warn policy, incomplete panel) — no \
7285 blocking findings from the seats that answered, verification green"
7286 )
7287 } else if incomplete {
7288 format!(
7289 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7290 quorum) — no blocking findings from the seats that answered, \
7291 verification green",
7292 expected - answered
7293 )
7294 } else {
7295 format!("round {round}: clean — no blocking findings, verification green")
7296 },
7297 );
7298 self.state.reviews.push(round_record);
7299 self.state.status = RunStatus::Gating;
7300 self.state.save()?;
7301 return Ok(());
7302 }
7303
7304 if incomplete && blocking == 0 && e2e_ok {
7312 self.state.reviews.push(round_record);
7313 self.state.save()?;
7314 if round == max_rounds {
7315 self.state.status = RunStatus::Blocked;
7316 self.state.event(
7317 "review",
7318 format!(
7319 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7320 refusing to call it clean",
7321 expected - answered
7322 ),
7323 );
7324 return Ok(());
7325 }
7326 continue;
7327 }
7328
7329 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7341 self.state.reviews.push(round_record);
7342 return self
7343 .stop_unresolved_review(
7344 "the round's own verification could not run",
7345 &shell,
7346 &winner.worktree,
7347 )
7348 .await;
7349 }
7350
7351 if round == max_rounds {
7352 self.state.reviews.push(round_record);
7353 return self
7354 .stop_reviewing(
7355 &format!(
7356 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7357 ),
7358 &shell,
7359 &winner.worktree,
7360 )
7361 .await;
7362 }
7363
7364 if !self.fix_review_round(&winner, round_record).await? {
7365 return Ok(());
7366 }
7367 }
7368 Ok(())
7369 }
7370
7371 async fn fix_review_round(
7374 &mut self,
7375 winner: &Candidate,
7376 mut round_record: ReviewRound,
7377 ) -> Result<bool> {
7378 let round = round_record.round;
7379 let shell = self.state.config.shell();
7380 let step = self.run_fix_round(winner, &mut round_record).await?;
7381 if matches!(step, FixStep::Resource) {
7382 self.stop_for_resource()?;
7383 return Ok(false);
7384 }
7385 if self.state.reviews.len() < round {
7386 self.state.reviews.push(round_record);
7387 } else {
7388 self.state.reviews[round - 1] = round_record;
7389 }
7390 self.state.save()?;
7391
7392 if matches!(step, FixStep::Recorded { report_lost: true }) {
7405 self.stop_reviewing(
7406 "the fixer's adoption report never came back, even after resuming its \
7407 own seat; refusing to start another round against the same worktree \
7408 while that is unresolved",
7409 &shell,
7410 &winner.worktree,
7411 )
7412 .await?;
7413 return Ok(false);
7414 }
7415
7416 let streak = self
7417 .state
7418 .reviews
7419 .iter()
7420 .rev()
7421 .take_while(|r| !r.progressed)
7422 .count();
7423 if streak >= STAGNANT_LIMIT {
7424 self.stop_reviewing(
7425 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7426 &shell,
7427 &winner.worktree,
7428 )
7429 .await?;
7430 return Ok(false);
7431 }
7432 Ok(true)
7433 }
7434
7435 async fn run_fix_round(
7437 &mut self,
7438 winner: &Candidate,
7439 round_record: &mut ReviewRound,
7440 ) -> Result<FixStep> {
7441 let round = round_record.round;
7442 let max_rounds = self.state.config.graph.review_rounds;
7443 let run_id = self.state.id.clone();
7444 let prompts = self.state.config.prompts.clone();
7445 let language = self.state.config.graph.language.clone();
7446 let sessions = self.state.config.graph.sessions;
7447 let artifacts = self.merge_repair_artifacts();
7448 let base = self.landing_base();
7449 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7450 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7451 let this_round_verification = round_record.verification_summary(&head);
7452 let all_findings: Vec<_> = round_record
7453 .reviews
7454 .iter()
7455 .flat_map(|r| r.findings.iter().cloned())
7456 .collect();
7457 let blocking_findings: Vec<_> = all_findings
7460 .iter()
7461 .filter(|f| f.severity.blocks())
7462 .cloned()
7463 .collect();
7464 let fix_prompt = prompt::fix_in(
7465 &self.state.instruction,
7466 &blocking_findings,
7467 this_round_verification.as_ref(),
7468 round,
7469 max_rounds,
7470 &language,
7471 &self.state.config.graph.github_language,
7472 );
7473 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7474 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7475 let (job, seat, out) = self
7476 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7477 prompt: fix_prompt.clone(),
7478 spec,
7479 seat,
7480 cwd: winner.worktree.clone(),
7481 timeout,
7482 allow_write: true,
7483 sessions,
7484 artifacts: artifacts.clone(),
7485 stem: format!("fix-{round}"),
7486 handover: None,
7487 })
7488 .await;
7489 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7492 return Ok(FixStep::Resource);
7493 }
7494 let agent_id = seat.agent.clone();
7495
7496 let mut fix = FixRecord {
7497 agent: agent_id,
7498 addressed: Vec::new(),
7499 rejected: Vec::new(),
7500 notes: String::new(),
7501 committed: false,
7502 failed: None,
7503 duration_ms: 0,
7504 continuation: None,
7505 };
7506 let mut continuation = ContinuationRecord::not_needed();
7507 let mut final_seat = seat.clone();
7508 match out {
7509 AgentOutcome::Ok(o) => {
7510 fix.duration_ms = o.duration_ms;
7511 let parsed = verdict::extract_json::<FixReport>(&o.text);
7512 let incomplete_reason = match &parsed {
7519 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7520 "the reply parsed, but it reported a command whose own CLI never \
7521 confirmed an exit status"
7522 .to_owned(),
7523 ),
7524 Ok(_) => None,
7525 Err(e) => Some(e.to_string()),
7526 };
7527 match incomplete_reason {
7528 None => {
7529 let report = parsed.expect("checked Ok above");
7530 fix.addressed = report.addressed;
7531 fix.rejected = report.rejected;
7532 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7533 }
7534 Some(reason) => {
7535 let (resumed_seat, resolved, failure, cont) = self
7536 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7537 .await;
7538 fix.duration_ms += cont.cumulative_wait_ms;
7539 continuation = cont;
7540 final_seat = resumed_seat;
7541 match resolved {
7542 Some(report) => {
7543 fix.addressed = report.addressed;
7544 fix.rejected = report.rejected;
7545 fix.notes =
7546 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7547 }
7548 None => fix.failed = failure,
7549 }
7550 }
7551 }
7552 }
7553 AgentOutcome::Dropped(o) => {
7555 fix.duration_ms = o.duration_ms;
7556 let why = o
7557 .dropped
7558 .as_ref()
7559 .map(|d| d.why.as_str())
7560 .unwrap_or("the CLI ended the stream without delivering its answer");
7561 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7562 }
7563 AgentOutcome::Quota(o) => {
7564 self.state.quota.push(QuotaLoss::from_report(
7565 final_seat.key.clone(),
7566 "fix",
7567 o.quota.as_ref(),
7568 ));
7569 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7570 }
7571 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7572 }
7573 if self.state.resource_block.is_some() {
7574 self.state.seats.insert(final_seat.key.clone(), final_seat);
7575 return Ok(FixStep::Resource);
7576 }
7577 fix.continuation = Some(continuation);
7578 self.state.seats.insert(final_seat.key.clone(), final_seat);
7579 let coordination = self.coordinate(&winner.worktree).await?;
7580 if let Ok(r) = git::rescue_commit(
7581 &winner.worktree,
7582 &format!("magi: review round {round} fixes (uncommitted work)"),
7583 )
7584 .await
7585 {
7586 self.state.note_withheld("fix", &r.withheld);
7587 }
7588 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7589 fix.committed = after != before;
7590 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7598 drop(coordination);
7599 let progressed = diff_after != patch;
7600 let commit_note = if fix.committed {
7601 "committed"
7602 } else {
7603 "NO new commit"
7604 };
7605 let tree_note = if progressed {
7606 "changed vs base"
7607 } else {
7608 "unchanged vs base"
7609 };
7610 self.state.event(
7611 "fix",
7612 match &fix.failed {
7613 Some(reason) => {
7619 format!(
7620 "round {round}: fixer's adoption report was lost ({reason}); \
7621 {commit_note}, tree {tree_note}"
7622 )
7623 }
7624 None => format!(
7625 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7626 {tree_note}{}",
7627 fix.addressed.len(),
7628 fix.rejected.len(),
7629 if continuation.outcome == ContinuationOutcome::Resumed {
7630 format!(
7631 " (adoption report recovered after {} continuation(s))",
7632 continuation.attempts
7633 )
7634 } else {
7635 String::new()
7636 },
7637 ),
7638 },
7639 );
7640 round_record.fix = Some(fix);
7641 round_record.progressed = progressed;
7642 Ok(FixStep::Recorded {
7643 report_lost: matches!(
7644 continuation.outcome,
7645 ContinuationOutcome::Exhausted
7646 | ContinuationOutcome::QuotaLost
7647 | ContinuationOutcome::NoSession
7648 ),
7649 })
7650 }
7651
7652 fn record_contested_handoff(&mut self) {
7686 if self.state.contested_handoff.is_some() {
7687 return;
7688 }
7689 let Some(contested) = self
7690 .state
7691 .reviews
7692 .last()
7693 .and_then(ReviewRound::contested_handoff)
7694 else {
7695 return;
7696 };
7697 self.state.event(
7698 "review",
7699 format!(
7700 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7701 wait for the owner's approval",
7702 contested.findings.len(),
7703 contested.rejecters.len()
7704 ),
7705 );
7706 self.state.contested_handoff = Some(contested);
7707 }
7708
7709 async fn stop_unresolved_review(
7710 &mut self,
7711 why: &str,
7712 shell: &[String],
7713 worktree: &Path,
7714 ) -> Result<()> {
7715 self.stop_reviewing_inner(why, shell, worktree, true).await
7716 }
7717
7718 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7719 self.stop_reviewing_inner(why, shell, worktree, false).await
7720 }
7721
7722 async fn stop_reviewing_inner(
7723 &mut self,
7724 why: &str,
7725 shell: &[String],
7726 worktree: &Path,
7727 repairable: bool,
7728 ) -> Result<()> {
7729 let round_idx = self.state.reviews.len() - 1;
7730 let needs_catchup_run = matches!(
7738 self.state.reviews[round_idx].e2e_status(),
7739 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7740 );
7741 if needs_catchup_run {
7742 let round = self.state.reviews[round_idx].round;
7743 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7744 let commands = self.state.config.verify.e2e.clone();
7745 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7746 let cache_dir = self.state.config.cache_dir();
7747 let context = format!(
7748 "round {round}: verification unresolved, catching up before the final decision"
7749 );
7750 let wait = self.repository_wait();
7751 let (outcomes, verify_retried) = with_cache_lease(
7752 &mut self.state,
7753 cache_dir.as_deref(),
7754 "e2e",
7755 "e2e",
7756 worktree,
7757 &attempted_head,
7758 timeout,
7759 &context,
7760 &wait,
7761 |state, budget| {
7762 let shell = shell.to_vec();
7763 let commands = commands.clone();
7764 let context = context.clone();
7765 async move {
7766 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7767 .await
7768 }
7769 },
7770 )
7771 .await?;
7772 let last = &mut self.state.reviews[round_idx];
7773 last.e2e = outcomes;
7774 last.verify_retried = verify_retried;
7775 last.verified_head = Some(attempted_head);
7782 last.verified_at = Some(Timestamp::now());
7783 if verify_inconclusive(&last.e2e) {
7784 self.state.save()?;
7791 return Ok(());
7792 }
7793 last.e2e_deferred = false;
7794 if last.answered == last.expected {
7799 last.clean = round_is_clean(
7800 last.blocking,
7801 last.e2e.iter().all(CommandOutcome::ok),
7802 last.answered,
7803 last.expected,
7804 0,
7805 self.state.config.graph.incomplete_review,
7806 );
7807 }
7808 }
7809 let last = &self.state.reviews[round_idx];
7810 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7811
7812 match last.e2e_status() {
7813 E2eStatus::Failed => {
7814 let red: Vec<String> = last
7815 .e2e
7816 .iter()
7817 .filter(|o| !o.ok())
7818 .map(|o| {
7819 format!(
7820 "`{}` -> {:?}\n{}",
7821 o.command,
7822 o.code,
7823 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7824 )
7825 })
7826 .collect();
7827 let can_repair = last.blocking == 0 && last.fix.is_none();
7828 self.state.reviews[round_idx].clean = false;
7829 self.state
7830 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7831 self.state.status = RunStatus::Blocked;
7832 if repairable
7833 && can_repair
7834 && self.state.reviews.len() < self.state.config.graph.review_rounds
7835 {
7836 self.state.status = RunStatus::Reviewing;
7837 }
7838 }
7839 E2eStatus::ResourceBlocked => {
7844 self.state.event(
7845 "review",
7846 format!(
7847 "{why}; e2e could not run (resource unavailable); not \
7848 deciding yet"
7849 ),
7850 );
7851 }
7852 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7853 self.state.event(
7854 "review",
7855 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7856 );
7857 self.record_contested_handoff();
7858 self.state.status = RunStatus::Gating;
7859 }
7860 }
7861 self.state.save()?;
7862 Ok(())
7863 }
7864
7865 async fn gate(&mut self) -> Result<()> {
7868 if self.hold_base_sync().await? {
7869 return Ok(());
7870 }
7871 if self.state.status == RunStatus::Failed
7883 || self
7884 .state
7885 .base_sync
7886 .as_ref()
7887 .is_some_and(|s| s.conflict.is_some())
7888 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7889 != Some(RunStatus::Gating)
7890 {
7891 return Ok(());
7892 }
7893 let Some(winner) = self.state.winner().cloned() else {
7894 return Ok(());
7895 };
7896 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7897 self.state.reconcile_verification_head(&head);
7898 self.state.save()?;
7899 let wait = self.repository_wait();
7900 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7901 return Ok(());
7902 }
7903 if self.state.gate_ran {
7904 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7915 self.state.status = RunStatus::Blocked;
7916 self.state.save()?;
7917 }
7918 return Ok(());
7919 }
7920 self.state.status = RunStatus::Gating;
7921 let mut outcomes = self.run_gate(&winner).await?;
7922 let mut attempted_head = self.state.gate_head.clone();
7923 loop {
7924 if verify_inconclusive(&outcomes) {
7935 self.state.save()?;
7936 return Ok(());
7937 }
7938 if outcomes.iter().all(CommandOutcome::ok) {
7939 break;
7940 }
7941 let fixes_before = self.state.gate_fixes.len();
7942 match self.gate_fix_round(&winner, &outcomes).await? {
7943 GateFix::Retry => {
7944 outcomes = self.run_gate(&winner).await?;
7945 attempted_head = self.state.gate_head.clone();
7946 }
7947 GateFix::Stop => {
7948 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7949 if self.state.gate_fixes.len() > fixes_before
7950 && attempted_head.as_deref() != Some(current.as_str())
7951 && verify_current_head(&mut self.state, &winner, &wait).await?
7952 {
7953 outcomes = self.run_gate(&winner).await?;
7956 attempted_head = self.state.gate_head.clone();
7957 } else {
7958 break;
7959 }
7960 }
7961 GateFix::Defer => {
7962 self.state.save()?;
7963 return Ok(());
7964 }
7965 }
7966 }
7967 let passed = outcomes.iter().all(CommandOutcome::ok);
7968 self.state.gate = outcomes;
7969 self.state.gate_ran = true;
7970 self.state.gate_head = attempted_head;
7971 self.state.save()?;
7972 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7973 return Ok(());
7974 }
7975 if !passed {
7976 self.state.status = RunStatus::Blocked;
7977 let spent = self.state.gate_fixes.len();
7978 self.state.event(
7979 "gate",
7980 if spent == 0 {
7981 "gate failed; not merging".to_owned()
7982 } else {
7983 format!("gate failed after {spent} gate-fix round(s); not merging")
7984 },
7985 );
7986 }
7987 self.state.save()?;
7988 Ok(())
7989 }
7990
7991 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8001 let commands = state.config.verify.pre_gate.clone();
8002 if commands.is_empty() {
8003 return;
8004 }
8005 let shell = state.config.shell();
8006 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8007 let (outcomes, _) = run_commands(
8008 state,
8009 "pre_gate",
8010 "pre_gate",
8011 0,
8012 &shell,
8013 &commands,
8014 &winner.worktree,
8015 timeout,
8016 )
8017 .await;
8018 for o in &outcomes {
8019 if !o.ok() {
8020 tracing::warn!(
8021 "pre_gate `{}` failed ({:?}); the gate decides",
8022 o.command,
8023 o.code
8024 );
8025 }
8026 state.event(
8027 "pre_gate",
8028 format!(
8029 "`{}` -> {}",
8030 o.command,
8031 if o.ok() {
8032 "pass".to_owned()
8033 } else {
8034 format!(
8035 "FAIL ({:?})\n{}",
8036 o.code,
8037 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8038 )
8039 }
8040 ),
8041 );
8042 }
8043 state.pre_gate = outcomes;
8044 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8045 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8046 Ok(head) => {
8047 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8048 state.pre_gate_commit = Some(head);
8049 }
8050 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8051 },
8052 Ok(false) => {}
8053 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8054 }
8055 if let Err(e) = state.save() {
8056 tracing::warn!("could not persist the pre_gate record: {e:#}");
8057 }
8058 }
8059
8060 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8063 let wait = self.repository_wait();
8064 run_landing_gate(&mut self.state, winner, &wait).await
8065 }
8066
8067 async fn gate_fix_round(
8080 &mut self,
8081 winner: &Candidate,
8082 outcomes: &[CommandOutcome],
8083 ) -> Result<GateFix> {
8084 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8085 g.phase == crate::repair_budget::Phase::Fixing
8086 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8087 }) {
8088 self.state.gate_fixes.len().saturating_add(1)
8089 } else {
8090 self.state.config.graph.gate_fix_rounds
8091 };
8092 let spent = self.state.gate_fixes.len();
8093 if spent >= cap {
8094 if cap > 0 {
8095 self.state.event(
8096 "gate",
8097 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8098 );
8099 }
8100 return Ok(GateFix::Stop);
8101 }
8102 if !gate_fixable(outcomes) {
8103 self.state.event(
8104 "gate",
8105 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8106 command or similar); not spending a fix round on it",
8107 );
8108 return Ok(GateFix::Stop);
8109 }
8110 let min_free = self.state.config.disk.min_free_bytes;
8111 if min_free > 0 {
8112 match crate::disk::free_bytes(&winner.worktree) {
8113 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8114 Ok(free) => {
8115 self.state.event(
8116 "gate",
8117 format!(
8118 "only {free} bytes free ({min_free} required by `[disk] \
8119 min_free_bytes`); not spending a fix round on a failure the disk \
8120 may explain"
8121 ),
8122 );
8123 return Ok(GateFix::Stop);
8124 }
8125 Err(e) => {
8126 self.state.event(
8127 "gate",
8128 format!("free disk space could not be measured ({e:#}); no fix round"),
8129 );
8130 return Ok(GateFix::Stop);
8131 }
8132 }
8133 }
8134
8135 let attempt = spent + 1;
8136 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8137 let base = self.landing_base();
8138 let fix_prompt = prompt::gate_fix_in(
8139 &self.state.instruction,
8140 &failed,
8141 attempt,
8142 cap,
8143 &self.state.config.graph.language,
8144 &self.state.config.graph.github_language,
8145 );
8146 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8147 let sessions = self.state.config.graph.sessions;
8148 let artifacts = self.merge_repair_artifacts();
8149 self.state.event(
8150 "gate",
8151 format!("gate failed; gate-fix round {attempt} of {cap}"),
8152 );
8153 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8154 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8155 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8158 head: before.clone(),
8159 passed: false,
8160 outcomes: Vec::new(),
8161 });
8162 self.state.save()?;
8163 let (_, seat, out) = self
8164 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8165 prompt: fix_prompt.clone(),
8166 spec,
8167 seat,
8168 cwd: winner.worktree.clone(),
8169 timeout,
8170 allow_write: true,
8171 sessions,
8172 artifacts: artifacts.clone(),
8173 stem: format!("gate-fix-{attempt}"),
8174 handover: None,
8175 })
8176 .await;
8177 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8180 return Ok(GateFix::Defer);
8181 }
8182 let mut record = GateFixRecord {
8183 agent: seat.agent.clone(),
8184 failed,
8185 notes: String::new(),
8186 committed: false,
8187 error: None,
8188 };
8189 match out {
8190 AgentOutcome::Ok(o) => {
8191 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8194 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8195 }
8196 }
8197 AgentOutcome::Dropped(_) => {
8198 record.error = Some("the CLI dropped the stream".to_owned());
8199 }
8200 AgentOutcome::Quota(o) => {
8201 self.state.quota.push(QuotaLoss::from_report(
8202 seat.key.clone(),
8203 "gate-fix",
8204 o.quota.as_ref(),
8205 ));
8206 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8207 }
8208 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8209 }
8210 self.state.seats.insert(seat.key.clone(), seat);
8211 let coordination = self.coordinate(&winner.worktree).await?;
8212 if let Ok(r) = git::rescue_commit(
8213 &winner.worktree,
8214 &format!("magi: gate fix {attempt} (uncommitted work)"),
8215 )
8216 .await
8217 {
8218 self.state.note_withheld("gate-fix", &r.withheld);
8219 }
8220 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8221 record.committed = after != before;
8222 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8223 let note = record.error.clone();
8224 self.state.gate_fixes.push(record);
8225 self.state.save()?;
8226 if !changed {
8227 self.state.event(
8228 "gate",
8229 match note {
8230 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8231 None => format!("gate-fix round {attempt}: the tree did not change"),
8232 },
8233 );
8234 return Ok(GateFix::Stop);
8235 }
8236 self.state.event(
8237 "gate",
8238 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8239 );
8240
8241 drop(coordination);
8242 let wait = self.repository_wait();
8243 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8244 }
8245
8246 async fn guarded_pr_message(
8253 &mut self,
8254 winner: &Candidate,
8255 facts: Option<&BranchFacts>,
8256 posting: bool,
8257 ) -> Result<Option<PrMessage>> {
8258 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8259 let head = match final_head.as_deref() {
8260 Some(h) => crate::pr_outcome::FinalHead::At(h),
8261 None => crate::pr_outcome::FinalHead::Unknown,
8262 };
8263 let summarized = if posting {
8265 Some(self.summarize_change(winner, final_head.as_deref()).await)
8266 } else {
8267 None
8268 };
8269 if self.state.resource_block.is_some() {
8274 self.state.event(
8275 "pr-summary",
8276 "a shared resource was busy; nothing was published, resume to summarize again",
8277 );
8278 return Ok(None);
8279 }
8280 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8281 anyhow::bail!(
8282 "the final commit moved during public description generation; retry before publishing"
8283 );
8284 }
8285 let note = match (&summarized, &self.state.change_summary) {
8286 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8287 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8288 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8289 (None, _) => ChangeNote::NotAsked,
8290 };
8291 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8292 if !posting {
8293 let identity = crate::scrub::Identity::current();
8294 return Ok(Some(PrMessage {
8295 title: crate::scrub::scrub(&pr.title, &identity),
8296 body: crate::scrub::scrub(&pr.body, &identity),
8297 }));
8298 }
8299 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8302 if let Some(known) = self
8303 .state
8304 .github_text
8305 .clone()
8306 .filter(|g| g.fingerprint == raw_fp)
8307 {
8308 return self.settle_withheld(winner, &pr, known).await;
8309 }
8310 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8311 let mut violations =
8312 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8313 let identity = crate::scrub::Identity::current();
8314 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8315 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8316 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8317 {
8318 violations.push(crate::github_text::Violation::SensitiveData);
8319 }
8320 let has_language = violations
8324 .iter()
8325 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8326 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8327 self.state.event(
8328 "github-text",
8329 if has_language {
8330 format!("description rejected: {violations:?}; requesting one rewrite")
8331 } else {
8332 format!("description rejected: {violations:?}; asking the owner")
8333 },
8334 );
8335 let mut rewritten = false;
8336 let mut rejected_rewrite = None;
8337 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8338 {
8339 let output = text;
8340 if let Some(title) = summary_title(&output) {
8341 let raw_body = summary_without_title(&output);
8344 let clean_title = crate::scrub::scrub(&title, &identity);
8345 let mut body = crate::scrub::scrub(&raw_body, &identity);
8346 let redacted = clean_title != title || body != raw_body;
8347 let title = clean_title;
8348 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8349 if !complete {
8350 self.state.event(
8351 "github-text",
8352 "description rewrite omitted required findings",
8353 );
8354 }
8355 if !body.trim().is_empty() {
8356 body.push_str("\n\n");
8357 body.push_str(&public_pr_evidence(&self.state, head));
8358 }
8359 let marker = format!("magi:run/{}", self.state.id);
8360 if !body.trim().is_empty() && !body.contains(&marker) {
8361 body.push_str(&format!("\n\n{marker}"));
8362 }
8363 let rewrite = PrMessage {
8364 title: title.clone(),
8365 body: body.clone(),
8366 };
8367 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8368 let mut failed = crate::github_text::check_for(
8369 &self.state.config.graph,
8370 &title,
8371 &body,
8372 redecision,
8373 );
8374 if (!crate::github_text::shareable(&title)
8375 || !crate::github_text::shareable(&body))
8376 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8377 {
8378 failed.push(crate::github_text::Violation::SensitiveData);
8379 }
8380 if complete
8381 && !title.trim().is_empty()
8382 && !body.trim().is_empty()
8383 && !failed.is_empty()
8384 {
8385 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8386 title: title.clone(),
8387 body: body.clone(),
8388 failed: failed
8389 .iter()
8390 .map(|v| crate::github_text::category(*v).to_owned())
8391 .collect(),
8392 });
8393 }
8394 if complete
8395 && !title.trim().is_empty()
8396 && !raw_body.trim().is_empty()
8397 && failed.is_empty()
8398 {
8399 decision = redecision;
8400 pr = PrMessage { title, body };
8401 rewritten = true;
8402 self.state
8403 .event("github-text", "description rewrite passed");
8404 if redacted {
8405 self.state.event(
8406 "github-text",
8407 "sensitive data removed from description rewrite",
8408 );
8409 }
8410 }
8411 }
8412 }
8413 if !rewritten {
8414 self.state.event(
8415 "github-text",
8416 "description rewrite unavailable or rejected; asking the owner",
8417 );
8418 if let Some(w) =
8419 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8420 {
8421 self.retire_github_text_question("the withheld text changed");
8424 let record = crate::run::GithubTextAsk {
8425 rewrite: rejected_rewrite,
8426 fingerprint: raw_fp,
8427 title: w.title,
8428 body: w.body,
8429 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8430 question: None,
8431 asks: 0,
8432 resolved: false,
8433 chosen_title: None,
8434 choice: None,
8435 };
8436 return self.settle_withheld(winner, &pr, record).await;
8437 }
8438 }
8439 }
8440 let (title, body) =
8441 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8442 Ok(Some(PrMessage { title, body }))
8443 }
8444
8445 async fn summarize_change(
8456 &mut self,
8457 winner: &Candidate,
8458 head: Option<&str>,
8459 ) -> std::result::Result<(), String> {
8460 let Some(head) = head else {
8461 return Err("the final commit could not be read".to_owned());
8462 };
8463 let remote = self.state.config.merge.remote.clone();
8464 let tracking = format!("{remote}/{}", self.state.base_branch);
8465 if !git::rev_exists(&winner.worktree, &tracking).await {
8466 return Err(format!("the remote base `{tracking}` could not be read"));
8467 }
8468 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8469 Ok(b) if !b.is_empty() => b,
8470 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8471 };
8472 let evidence = crate::pr_outcome::summary_evidence(
8473 &self.state,
8474 crate::pr_outcome::FinalHead::At(head),
8475 );
8476 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8477 if self
8478 .state
8479 .change_summary
8480 .as_ref()
8481 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8482 {
8483 return Ok(());
8484 }
8485 if self.state.change_summary.take().is_some() {
8486 self.state.event(
8487 "pr-summary",
8488 "the final diff or evidence changed; the earlier change summary was discarded",
8489 );
8490 }
8491 if let Err(e) = write_artifact(
8493 &self.state,
8494 &format!("public-description-{fingerprint}.md"),
8495 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8496 ) {
8497 tracing::warn!("could not save public description evidence: {e:#}");
8498 }
8499 let chain = match agent::pick_chain(
8500 &self.state.config.agents,
8501 self.state.config.roles.synthesizer.as_ref(),
8502 &agent::installed,
8503 "synthesizer",
8504 ) {
8505 Ok(chain) => chain,
8506 Err(e) => {
8507 tracing::warn!("no agent for the change summary: {e:#}");
8508 self.state.event("pr-summary", "no summary agent available");
8509 return Err("no agent was available to summarize it".to_owned());
8510 }
8511 };
8512 let subjects = git::subjects(&winner.worktree, &base, head)
8513 .await
8514 .unwrap_or_default();
8515 let stat = git::diff_stat(&winner.worktree, &base, head)
8516 .await
8517 .unwrap_or_default();
8518 let stat: String = stat.chars().take(6_000).collect();
8519 let mut prompt = crate::prompt::change_summary_in(
8520 &base,
8521 head,
8522 &subjects,
8523 stat.trim(),
8524 &self.state.config.graph.github_language,
8525 );
8526 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8527 let sessions = self.state.config.graph.sessions;
8528 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8529 let prompts = self.state.config.prompts.clone();
8530 let cache = self.state.config.cache_dir();
8531 let run = self.state.id.clone();
8532 for (i, spec) in chain.into_iter().enumerate() {
8533 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8534 let github_spec = match agent::for_github_text(
8535 &spec,
8536 &self.state.config.graph.github_language,
8537 &winner.worktree,
8538 ) {
8539 Ok(spec) => spec,
8540 Err(e) => {
8541 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8542 continue;
8543 }
8544 };
8545 let job = SeatJob {
8546 spec: github_spec,
8547 seat,
8548 cwd: winner.worktree.clone(),
8549 prompt: prompt.clone(),
8550 timeout,
8551 allow_write: false,
8552 sessions,
8553 artifacts: agent::artifacts_dir(&self.state.dir()),
8554 stem: if i == 0 {
8555 "change-summary".to_owned()
8556 } else {
8557 format!("change-summary-{}", spec.id)
8558 },
8559 handover: None,
8560 };
8561 let ctx = WaveCtx {
8562 run: &run,
8563 node: "pr-summary",
8564 prompts: &prompts,
8565 cache: cache.as_deref(),
8566 round: None,
8567 carry_seats: false,
8568 pauses: Some((&self.pause, &self.interrupt)),
8569 };
8570 let (seat, outcome) =
8571 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8572 self.state.seats.insert(seat.key.clone(), seat);
8573 let text = match outcome {
8574 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8575 AgentOutcome::ResourceBlocked(why) => {
8576 if self.state.resource_block.is_none() {
8579 self.state.resource_block = Some(crate::run::ResourceBlock {
8580 cause: crate::run::ResourceBlockCause::Other,
8581 node: "pr-summary".to_owned(),
8582 kind: "shared resource".to_owned(),
8583 detail: why,
8584 at: Timestamp::now(),
8585 });
8586 }
8587 return Err("a shared resource was busy".to_owned());
8588 }
8589 _ => continue,
8590 };
8591 let blind = &self.state.config.blind;
8592 let Some(title) = summary_title(&text) else {
8593 continue;
8594 };
8595 let body: String = summary_without_title(&text)
8596 .lines()
8597 .filter(|l| !l.contains("magi:run/"))
8598 .collect::<Vec<_>>()
8599 .join("\n");
8600 let body = blind::sanitize_prose(body.trim(), blind);
8601 let title = blind::sanitize_prose(&title, blind);
8602 if body.trim().is_empty() || title.trim().is_empty() {
8603 continue;
8604 }
8605 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8606 continue;
8607 }
8608 if git::rev_parse(&winner.worktree, "HEAD")
8610 .await
8611 .ok()
8612 .as_deref()
8613 != Some(head)
8614 {
8615 return Err("the branch moved while the summary was written".to_owned());
8616 }
8617 if git::merge_base(&winner.worktree, &tracking, head)
8618 .await
8619 .ok()
8620 .as_deref()
8621 != Some(base.as_str())
8622 || crate::pr_outcome::summary_evidence(
8623 &self.state,
8624 crate::pr_outcome::FinalHead::At(head),
8625 ) != evidence
8626 {
8627 return Err("the summary evidence changed while it was written".to_owned());
8628 }
8629 self.state.change_summary = Some(crate::run::ChangeSummary {
8630 head: head.to_owned(),
8631 base,
8632 evidence: fingerprint,
8633 title,
8634 body,
8635 });
8636 self.state.event(
8637 "pr-summary",
8638 format!("change summary written by {} for the final diff", spec.id),
8639 );
8640 if let Err(e) = self.state.save() {
8641 tracing::warn!("could not save the change summary: {e:#}");
8642 }
8643 return Ok(());
8644 }
8645 self.state
8646 .event("pr-summary", "no agent produced a change summary");
8647 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8648 }
8649
8650 async fn rewrite_pr_text(
8660 &mut self,
8661 winner: &Candidate,
8662 pr: &PrMessage,
8663 violations: &[crate::github_text::Violation],
8664 ) -> Option<String> {
8665 let prompt = format!(
8666 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8667 pr.title, pr.body
8668 );
8669 let sessions = self.state.config.graph.sessions;
8670 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8671 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8674 let key = format!("impl-{}", winner.label);
8675 if let Some(spec) = self
8676 .state
8677 .config
8678 .agents
8679 .iter()
8680 .find(|a| a.id == winner.agent)
8681 .cloned()
8682 && let Some(existing) = self.state.seats.get(&key)
8683 && existing.agent == spec.id
8684 && agent::has_session(spec.kind, existing, sessions)
8685 {
8686 attempts.push((spec, Some(existing.clone())));
8687 }
8688 match agent::pick_chain(
8689 &self.state.config.agents,
8690 self.state.config.roles.language_judge.as_ref(),
8691 &agent::installed,
8692 "language_judge",
8693 ) {
8694 Ok(chain) => {
8695 for spec in chain {
8696 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8697 continue;
8698 }
8699 attempts.push((spec, None));
8700 }
8701 }
8702 Err(e) if attempts.is_empty() => {
8703 tracing::warn!("no agent for the description rewrite: {e:#}");
8704 self.state
8705 .event("github-text", "no rewrite agent available");
8706 return None;
8707 }
8708 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8709 }
8710 let prompts = self.state.config.prompts.clone();
8711 let cache = self.state.config.cache_dir();
8712 let run = self.state.id.clone();
8713 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8714 let resumed = resumed_seat.is_some();
8715 let seat = resumed_seat.unwrap_or_else(|| {
8716 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8717 });
8718 let github_spec = match agent::for_github_text(
8719 &spec,
8720 &self.state.config.graph.github_language,
8721 &winner.worktree,
8722 ) {
8723 Ok(spec) => spec,
8724 Err(e) => {
8725 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8726 continue;
8727 }
8728 };
8729 let job = SeatJob {
8730 spec: github_spec,
8731 seat,
8732 cwd: winner.worktree.clone(),
8733 prompt: prompt.clone(),
8734 timeout: retry_budget(timeout_full, resumed),
8735 allow_write: false,
8736 sessions,
8737 artifacts: agent::artifacts_dir(&self.state.dir()),
8738 stem: if i == 0 {
8739 "github-text-rewrite".to_owned()
8740 } else {
8741 format!("github-text-rewrite-{}", spec.id)
8742 },
8743 handover: None,
8744 };
8745 let ctx = WaveCtx {
8746 run: &run,
8747 node: "github-text",
8748 prompts: &prompts,
8749 cache: cache.as_deref(),
8750 round: None,
8751 carry_seats: false,
8752 pauses: Some((&self.pause, &self.interrupt)),
8753 };
8754 let (seat, outcome) =
8755 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8756 self.state.seats.insert(seat.key.clone(), seat);
8757 match outcome {
8758 AgentOutcome::Ok(o)
8759 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8760 {
8761 return Some(o.text);
8762 }
8763 AgentOutcome::ResourceBlocked(_) => return None,
8764 _ => {}
8765 }
8766 }
8767 None
8768 }
8769
8770 fn retire_github_text_question(&mut self, why: &str) {
8772 let Some(id) = self
8773 .state
8774 .github_text
8775 .as_ref()
8776 .and_then(|g| g.question.clone())
8777 else {
8778 return;
8779 };
8780 let _ = ask::Questions::open().update(&id, |q| {
8781 q.abandon(why);
8782 Ok(())
8783 });
8784 }
8785
8786 async fn settle_withheld(
8790 &mut self,
8791 winner: &Candidate,
8792 pr: &PrMessage,
8793 mut g: crate::run::GithubTextAsk,
8794 ) -> Result<Option<PrMessage>> {
8795 use crate::github_text as gt;
8796 let store = ask::Questions::open();
8797 let timeout = self.state.config.graph.answer_timeout;
8798 if !g.resolved {
8799 let mut standing = g
8800 .question
8801 .as_ref()
8802 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8803 if let Some(q) = standing.as_ref()
8804 && q.status.open()
8805 && gt::expired(q, timeout)
8806 {
8807 let why = format!("no answer within {}s of asking", timeout.max(1));
8808 standing = store
8809 .update(&q.id, |q| {
8810 q.abandon(&why);
8811 Ok(())
8812 })
8813 .ok()
8814 .map(|(q, ())| q);
8815 }
8816 let mut ask_again: Option<Option<Vec<String>>> = None;
8817 match standing {
8818 None if g.asks <= 1 => {
8821 g.asks = 0;
8822 ask_again = Some(None);
8823 }
8824 None => {
8825 g.resolved = true;
8826 g.chosen_title = None;
8827 g.choice = Some(GithubTextChoice::Fallback);
8828 }
8829 Some(q) if q.status.open() => {
8830 self.park_on_github_text(&g)?;
8831 return Ok(None);
8832 }
8833 Some(q) => match gt::read_reply(&q) {
8834 gt::Reply::Fallback => {
8835 g.resolved = true;
8836 g.chosen_title = None;
8837 g.choice = Some(GithubTextChoice::Fallback);
8838 }
8839 gt::Reply::Original => {
8840 g.resolved = true;
8841 g.chosen_title = None;
8842 g.choice = Some(GithubTextChoice::Original);
8843 }
8844 gt::Reply::Rewrite => {
8845 g.resolved = true;
8846 g.chosen_title = None;
8847 g.choice = Some(if g.rewrite.is_some() {
8848 GithubTextChoice::Rewrite
8849 } else {
8850 GithubTextChoice::Fallback
8851 });
8852 }
8853 gt::Reply::Title(text) => {
8854 let first = text
8855 .lines()
8856 .map(str::trim)
8857 .find(|l| !l.is_empty())
8858 .unwrap_or("");
8859 let decision = if first.is_empty() {
8860 None
8861 } else {
8862 let candidate = PrMessage {
8863 title: first.to_owned(),
8864 body: String::new(),
8865 };
8866 self.judge_pr_language(&winner.worktree, &candidate).await
8867 };
8868 match gt::vet_title(
8869 &text,
8870 decision,
8871 self.state.config.graph.github_english(),
8872 ) {
8873 Ok(title) => {
8874 g.resolved = true;
8875 g.chosen_title = Some(title);
8876 g.choice = Some(GithubTextChoice::MyTitle);
8877 }
8878 Err(cats) if g.asks < 2 => {
8879 ask_again = Some(Some(
8880 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8881 ))
8882 }
8883 Err(_) => {
8884 g.resolved = true;
8885 g.chosen_title = None;
8886 g.choice = Some(GithubTextChoice::Fallback);
8887 }
8888 }
8889 }
8890 },
8891 }
8892 if let Some(rejected) = ask_again {
8893 let w = gt::Withheld {
8896 title: g.title,
8897 body: g.body,
8898 categories: g.categories.clone(),
8899 };
8900 let artifact = match crate::run::write_artifact(
8903 &self.state,
8904 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8905 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8906 ) {
8907 Ok(p) => Some(p),
8908 Err(e) => {
8909 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8910 None
8911 }
8912 };
8913 let (detail, comparison) = gt::question_content(
8914 &w,
8915 &pr.title,
8916 &pr.body,
8917 &self.state.id,
8918 rejected.as_deref(),
8919 artifact.as_deref(),
8920 g.rewrite.as_ref(),
8921 );
8922 let mut q = ask::Question::new(
8923 self.state.id.clone(),
8924 gt::ASK_NODE.to_owned(),
8925 gt::ASK_SEAT.to_owned(),
8926 gt::question_summary(&self.state.config.graph.language, &w),
8927 detail,
8928 gt::question_choices(&w, g.rewrite.as_ref()),
8929 );
8930 q.posting_comparison = Some(comparison);
8931 q.answer_timeout = timeout;
8932 g.question = Some(q.id.clone());
8935 g.asks += 1;
8936 self.state.github_text = Some(g.clone());
8937 self.state.event(
8938 "github-text",
8939 format!("asking the owner about the withheld text ({})", q.short()),
8940 );
8941 self.state.save()?;
8942 store
8943 .put_executable(&mut q)
8944 .context("file the posting-gate question")?;
8945 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8946 tracing::warn!(
8947 "could not notify about posting-gate question {}: {e:#}",
8948 q.short()
8949 );
8950 }
8951 self.park_on_github_text(&g)?;
8952 return Ok(None);
8953 }
8954 }
8955 let resolved = g.chosen_title.clone();
8956 let original = g.choice == Some(GithubTextChoice::Original);
8957 self.state.event(
8959 "github-text",
8960 if original {
8961 format!(
8962 "posting the original text for: {}",
8963 match (g.title, g.body) {
8964 (true, true) => "title, description",
8965 (true, false) => "title",
8966 _ => "description",
8967 }
8968 )
8969 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8970 "posting the saved rewrite title and description".to_owned()
8971 } else if resolved.is_some() {
8972 "posting the owner's replacement title".to_owned()
8973 } else {
8974 "posting the neutral text for the withheld fields".to_owned()
8975 },
8976 );
8977 self.state.github_text = Some(g.clone());
8978 self.state.save()?;
8979 if g.choice == Some(GithubTextChoice::Rewrite)
8980 && let Some(r) = g.rewrite.as_ref()
8981 {
8982 return Ok(Some(PrMessage {
8983 title: r.title.clone(),
8984 body: r.body.clone(),
8985 }));
8986 }
8987 let w = gt::Withheld {
8988 title: g.title,
8989 body: g.body,
8990 categories: g.categories.clone(),
8991 };
8992 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8993 if let Some(t) = resolved {
8994 title = t;
8995 }
8996 let english = gt::LanguageDecision {
8998 title_english: true,
8999 body_english: true,
9000 body_complete: true,
9001 };
9002 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9003 if original {
9004 if g.title {
9007 title = pr.title.clone();
9008 }
9009 if g.body {
9010 body = pr.body.clone();
9011 }
9012 }
9013 Ok(Some(PrMessage { title, body }))
9014 }
9015
9016 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9017 self.state.github_text = Some(g.clone());
9018 self.state.end_pass_parked(ParkCause::Question);
9019 self.state.parked = true;
9020 self.state.event(
9021 "github-text",
9022 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9023 );
9024 self.state.save()
9025 }
9026
9027 async fn judge_pr_language(
9030 &mut self,
9031 cwd: &Path,
9032 pr: &PrMessage,
9033 ) -> Option<crate::github_text::LanguageDecision> {
9034 self.state.config.roles.language_judge.as_ref()?;
9035 if !self.state.config.graph.language_gate() {
9036 return None;
9037 }
9038 let decision =
9039 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9040 self.state.event(
9041 "github-text",
9042 if decision.is_some() {
9043 "language decided by the language judge"
9044 } else {
9045 "language judge unavailable; using built-in heuristics"
9046 },
9047 );
9048 decision
9049 }
9050
9051 async fn merge(&mut self) -> Result<()> {
9052 if self.resource_halt()? || self.hold_base_sync().await? {
9053 return Ok(());
9054 }
9055 if self
9070 .state
9071 .base_sync
9072 .as_ref()
9073 .is_some_and(|s| s.conflict.is_some())
9074 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9075 != Some(RunStatus::Gating)
9076 || !self.state.gate_allows_progress()
9084 {
9085 return Ok(());
9086 }
9087 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9096 return Ok(());
9097 }
9098 let Some(winner) = self.state.winner().cloned() else {
9099 return Ok(());
9100 };
9101 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9102 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9103 self.state.status = RunStatus::Blocked;
9104 self.state.event(
9105 "gate",
9106 "winner changed after gate; fresh verification required",
9107 );
9108 self.state.save()?;
9109 return Ok(());
9110 }
9111 let wait = self.repository_wait();
9112 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9113 return Ok(());
9114 }
9115 let repo = self.state.repo.clone();
9116 let base = self.state.base_branch.clone();
9117 let mode = self.state.config.merge.mode;
9118 let style = self.state.config.merge.style;
9119 let facts = if is_review_run(&self.state) {
9120 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9121 let start = review_base(
9122 &repo,
9123 &self.state.config.merge.remote,
9124 &base,
9125 &self.state.base_commit,
9126 &winner.branch,
9127 )
9128 .await;
9129 branch_facts(&repo, &start, &winner.branch).await
9130 } else {
9131 None
9132 };
9133 let leaked = if is_review_run(&self.state) {
9136 leaked_subjects(&self.state, &winner.branch).await
9137 } else {
9138 Some(Vec::new())
9139 };
9140 let Some(pr) = self
9144 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9145 .await?
9146 else {
9147 return Ok(());
9148 };
9149 let message = pr.commit_message();
9150
9151 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9152 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9153 return Ok(());
9154 }
9155 let outcome = match mode {
9156 MergeMode::None => MergeOutcome {
9157 mode,
9158 ok: true,
9159 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9160 empty: false,
9161 },
9162 MergeMode::Pr | MergeMode::Local
9163 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9164 {
9165 MergeOutcome {
9166 mode,
9167 ok: false,
9168 detail: empty_candidate_detail(&self.state, &base),
9169 empty: true,
9170 }
9171 }
9172 MergeMode::Local => {
9173 let on = git::current_branch(&repo).await?;
9174 if on.as_deref() != Some(base.as_str()) {
9175 MergeOutcome {
9176 mode,
9177 ok: false,
9178 detail: format!(
9179 "{} has {} checked out, not the base branch {base}",
9180 repo.display(),
9181 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9182 ),
9183 empty: false,
9184 }
9185 } else if !git::is_clean(&repo).await? {
9186 MergeOutcome {
9187 mode,
9188 ok: false,
9189 detail: format!("{} is dirty; refusing to merge", repo.display()),
9190 empty: false,
9191 }
9192 } else {
9193 let out = match style {
9194 MergeStyle::Merge => {
9195 git::merge_no_ff(&repo, &winner.branch, &message).await?
9196 }
9197 MergeStyle::Squash => {
9198 git::merge_squash(&repo, &winner.branch, &message).await?
9199 }
9200 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9201 };
9202 MergeOutcome {
9203 mode,
9204 ok: out.ok(),
9205 detail: if out.ok() { out.stdout } else { out.stderr },
9206 empty: false,
9207 }
9208 }
9209 }
9210 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9211 self.state.pr_publication_pending = true;
9212 self.state.save()?;
9213 let out =
9214 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9215 self.state.pr_publication_pending = out.is_err();
9216 MergeOutcome {
9217 mode,
9218 ok: out.is_ok(),
9219 detail: match out {
9220 Ok(url) => url,
9221 Err(e) => format!("{e:#}"),
9222 },
9223 empty: false,
9224 }
9225 }
9226 MergeMode::Pr => {
9227 let remote = self.state.config.merge.remote.clone();
9228 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9229 let pushed = if self.state.foreign_checkout.is_some() {
9230 let refspec = format!("{delivery_head}:refs/heads/{public}");
9231 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9232 } else {
9233 git::push(&winner.worktree, &remote, &winner.branch).await?
9234 };
9235 if !pushed.ok() {
9236 MergeOutcome {
9237 mode,
9238 ok: false,
9239 detail: pushed.stderr,
9240 empty: false,
9241 }
9242 } else {
9243 if self.state.foreign_checkout.is_some() {
9244 self.state.pr_pushed_head = Some(delivery_head.clone());
9245 }
9246 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9252 let out = match pr_merge_plan(found) {
9253 PrPlan::Create => {
9254 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9255 .await
9256 }
9257 PrPlan::Adopt { url, title } => {
9258 self.state
9259 .event("merge", format!("Pr: adopted open pull request {url}"));
9260 if self.state.github_text.as_ref().is_some_and(|g| {
9261 g.resolved
9262 && g.choice == Some(GithubTextChoice::Rewrite)
9263 && g.rewrite.is_some()
9264 }) {
9265 land::set_pr_text_verbatim(
9268 &winner.worktree,
9269 &url,
9270 &pr.title,
9271 &pr.body,
9272 )
9273 .await
9274 .map(|()| url)
9275 } else {
9276 if title != pr.title
9277 && (!is_review_run(&self.state)
9278 || leaked
9279 .as_deref()
9280 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9281 && let Err(e) =
9282 if self.state.github_text.as_ref().is_some_and(|g| {
9283 g.resolved
9284 && g.title
9285 && g.choice == Some(GithubTextChoice::Original)
9286 }) {
9287 land::set_pr_title_verbatim(
9289 &winner.worktree,
9290 &url,
9291 &pr.title,
9292 )
9293 .await
9294 } else {
9295 land::set_pr_title(
9296 &mut self.state,
9297 &winner.worktree,
9298 &url,
9299 &pr.title,
9300 )
9301 .await
9302 }
9303 {
9304 tracing::warn!("could not refresh title of {url}: {e:#}");
9305 self.state
9306 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9307 }
9308 Ok(url)
9309 }
9310 }
9311 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9312 };
9313 match out {
9314 Ok(url) => MergeOutcome {
9315 mode,
9316 ok: true,
9317 detail: url,
9318 empty: false,
9319 },
9320 Err(e) => MergeOutcome {
9321 mode,
9322 ok: false,
9323 detail: e.to_string(),
9324 empty: false,
9325 },
9326 }
9327 }
9328 }
9329 };
9330
9331 self.state.status = match (mode, outcome.ok) {
9332 (MergeMode::None, _) => RunStatus::Ready,
9333 (_, true) if outcome.empty => RunStatus::Ready,
9334 (MergeMode::Pr, true) => RunStatus::Ready,
9335 (_, true) => RunStatus::Merged,
9336 (_, false) => RunStatus::Blocked,
9337 };
9338 self.state.event(
9339 "merge",
9340 format!(
9341 "{:?}: {}",
9342 mode,
9343 outcome.detail.lines().next().unwrap_or("")
9344 ),
9345 );
9346 self.state.merge = Some(outcome);
9347 self.state.save()?;
9348
9349 if self.state.config.graph.land
9355 && mode == MergeMode::Pr
9356 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9357 {
9358 self.run_land().await?;
9359 }
9360 self.settle_questions();
9365 Ok(())
9366 }
9367
9368 async fn run_land(&mut self) -> Result<()> {
9379 crate::pr_head::ensure_owner(&self.state)?;
9380 let url = self
9381 .state
9382 .pr
9383 .as_ref()
9384 .map(|pr| pr.url.clone())
9385 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9386 .unwrap_or_default();
9387 let url = url.lines().next().unwrap_or("").trim().to_owned();
9388 if !url.starts_with("http") {
9389 return Ok(());
9390 }
9391 loop {
9394 match land::land(&mut self.state, &url).await {
9395 Ok(_)
9396 if self
9397 .state
9398 .merge_repairs
9399 .last()
9400 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9401 {
9402 if self.repair_merge_review().await? {
9403 continue;
9404 }
9405 }
9406 Ok(_) if self.state.resource_block.is_some() => {
9407 return self.stop_for_resource();
9408 }
9409 Ok(pr) if self.state.parked => {
9410 let _ = pr;
9414 }
9415 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9416 self.state.status = RunStatus::Superseded;
9417 self.state.save()?;
9418 }
9419 Ok(pr) => {
9420 self.state.status = match pr.state {
9421 land::PrLifecycle::Merged => RunStatus::Merged,
9422 _ => RunStatus::Blocked,
9423 };
9424 if bump::should_release_bump(self.state.status)
9431 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9432 {
9433 self.state
9439 .event("bump", format!("release bump skipped: {e:#}"));
9440 }
9441 if self.state.status == RunStatus::Merged {
9444 crate::followup::after_merge(&mut self.state, &pr.url).await;
9445 }
9446 self.state.save()?;
9447 }
9448 Err(e) => {
9449 if e.is::<crate::repo_lock::ResourceBlocked>() {
9450 return Err(e);
9451 }
9452 self.state.status = RunStatus::Blocked;
9453 self.state.event("land", format!("gave up: {e}"));
9454 self.state.save()?;
9455 }
9456 }
9457 break;
9458 }
9459 Ok(())
9460 }
9461
9462 async fn repair_merge_review(&mut self) -> Result<bool> {
9465 use crate::run::MergeRepairPhase as Phase;
9466 crate::pr_head::ensure_owner(&self.state)?;
9467 let i = self.state.merge_repairs.len() - 1;
9468 let request = self.state.merge_repairs[i].clone();
9469 if request.phase == Phase::Fixing {
9470 let restored = match self.state.winner().cloned() {
9471 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9472 None => Err(anyhow::anyhow!("repair has no winning branch")),
9473 };
9474 let why = match restored {
9475 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9476 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9477 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9478 };
9479 self.hold_merge_repair(i, why)?;
9480 return Ok(false);
9481 }
9482 if request.phase == Phase::Reviewing && self.state.gate_ran {
9487 let winner = self
9488 .state
9489 .winner()
9490 .context("repair has no winning branch")?;
9491 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9492 if self.state.gate_head.as_deref() != Some(head.as_str())
9493 || !git::is_clean(&winner.worktree).await?
9494 {
9495 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9496 return Ok(false);
9497 }
9498 }
9499 let failure = if i >= self.state.config.graph.land_rounds
9500 || self.state.config.graph.review_rounds == 0
9501 {
9502 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9503 } else {
9504 None
9505 };
9506 if let Some(why) = failure {
9507 self.hold_merge_repair(i, why)?;
9508 return Ok(false);
9509 }
9510 if request.phase == Phase::Requested {
9511 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9512 Ok(pr) => pr,
9513 Err(e) => {
9514 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9515 return Ok(false);
9516 }
9517 };
9518 self.state.merge_repairs[i].phase = Phase::Fixing;
9519 self.state.land_approval = None;
9520 self.state.land_armed_head = None;
9521 self.state.save()?;
9522 let evidence = serde_json::to_string_pretty(&request)?;
9525 let reason = format!(
9526 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9527 );
9528 let budget = self.state.config.graph.land_rounds;
9529 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9530 match result {
9531 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9532 Ok(land::Fixed::Deferred) => {
9533 let worktree = self
9537 .state
9538 .winner()
9539 .context("repair has no winning branch")?
9540 .worktree
9541 .clone();
9542 self.graft_restore(&worktree, &request.head).await?;
9543 self.state.merge_repairs[i].phase = Phase::Requested;
9544 self.stop_for_resource()?;
9545 return Ok(false);
9546 }
9547 Ok(land::Fixed::Failed(why)) => {
9548 self.hold_merge_repair(
9549 i,
9550 format!(
9551 "repair failed: {why}; inspect the branch and run an explicit review"
9552 ),
9553 )?;
9554 return Ok(false);
9555 }
9556 Err(e) => {
9557 self.hold_merge_repair(
9558 i,
9559 format!(
9560 "repair failed: {e:#}; inspect the branch and run an explicit review"
9561 ),
9562 )?;
9563 return Ok(false);
9564 }
9565 }
9566 self.begin_merge_repair_review(i)?;
9570 }
9571 self.review_loop().await?;
9572 self.gate().await?;
9573 if self.resource_halt()? || self.state.parked {
9574 return Ok(false);
9575 }
9576 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9577 != Some(RunStatus::Gating)
9578 || !self.state.gate_allows_progress()
9579 {
9580 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9581 return Ok(false);
9582 }
9583 let winner = self
9586 .state
9587 .winner()
9588 .cloned()
9589 .context("repair has no winning branch")?;
9590 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9591 if self.state.gate_head.as_deref() != Some(after.as_str())
9592 || !git::is_clean(&winner.worktree).await?
9593 {
9594 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9595 return Ok(false);
9596 }
9597 let expected = self
9598 .state
9599 .pr_pushed_head
9600 .as_deref()
9601 .unwrap_or(&request.head)
9602 .to_owned();
9603 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9604 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9607 return Ok(false);
9608 }
9609 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9610 let refspec = format!("{after}:refs/heads/{public}");
9611 let push = git::git_raw(
9612 &winner.worktree,
9613 &["push", &lease, &self.state.config.merge.remote, &refspec],
9614 )
9615 .await;
9616 match push {
9617 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9618 Ok(out) => {
9619 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9620 return Ok(false);
9621 }
9622 Err(e) => {
9623 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9624 return Ok(false);
9625 }
9626 }
9627 self.state.merge_repairs[i].phase = Phase::Complete;
9628 self.state.status = RunStatus::Landing;
9629 self.state.event(
9630 "repair",
9631 "new review and gate completed; fresh owner merge decision required",
9632 );
9633 self.state.save()?;
9634 Ok(true)
9635 }
9636
9637 fn merge_repair_artifacts(&self) -> PathBuf {
9638 let artifacts = agent::artifacts_dir(&self.state.dir());
9639 match self
9640 .state
9641 .merge_repairs
9642 .last()
9643 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9644 {
9645 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9646 None => artifacts,
9647 }
9648 }
9649
9650 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9651 if let Some(e2e) = self.state.head_e2e.take() {
9652 self.state
9653 .verification_history
9654 .push(crate::run::VerificationEvidence {
9655 superseded_by: None,
9656 archived_at: Some(Timestamp::now()),
9657 fixes_spent: Some(self.state.gate_fixes.len()),
9658 gate_head: None,
9659 gate_ran: false,
9660 gate: Vec::new(),
9661 e2e: Some(e2e),
9662 });
9663 }
9664 self.state.reviews.clear();
9665 self.state.gate.clear();
9666 self.state.gate_ran = false;
9667 self.state.gate_head = None;
9668 self.state.gate_fixes.clear();
9669 self.state.gate_repair_e2e = None;
9670 self.state.pre_gate.clear();
9671 self.state.pre_gate_commit = None;
9672 self.state.contested_handoff = None;
9673 self.state.land_reruns = None;
9674 self.state.land_approval = None;
9675 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9676 self.state.status = RunStatus::Reviewing;
9677 self.state.save()
9678 }
9679
9680 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9681 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9682 self.state.merge_repairs[index].failure = Some(why.clone());
9683 self.state.status = RunStatus::Blocked;
9684 self.state.event("repair", why);
9685 self.state.save()
9686 }
9687
9688 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9692 if let Some(existing) = self.state.seats.get(key)
9693 && existing.agent == agent
9694 {
9695 return existing.clone();
9696 }
9697 let fresh = if self.state.seats.contains_key(key) {
9700 handover_seat(key, agent, self.state.next_seat_seed())
9701 } else {
9702 SeatState::new(key, agent, self.state.seed)
9703 };
9704 self.state.seats.insert(key.to_owned(), fresh.clone());
9705 fresh
9706 }
9707
9708 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9714 match self.state.seats.get(key) {
9715 Some(s) if s.agent != spec.id => {
9716 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9717 }
9718 _ => spec,
9719 }
9720 }
9721
9722 fn view(&self, c: &Candidate) -> CandidateView {
9724 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9725 .unwrap_or_default();
9726 let (patch, _) = blind::sanitize_patch(
9727 &format!("candidate {} patch", c.label),
9728 &raw,
9729 &self.state.config.blind,
9730 );
9731 CandidateView {
9732 label: c.label,
9733 branch: c.branch.clone(),
9734 summary: c.summary.clone(),
9735 stat: c.stat.clone(),
9736 patch,
9737 }
9738 }
9739
9740 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9742 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9743 prompt::judge(
9744 "(see above)",
9745 &views,
9746 self.roles.judges.len(),
9747 base_short,
9748 "en",
9749 )
9750 }
9751
9752 fn vote_prompt_full(
9758 &self,
9759 j: usize,
9760 labels: &[char],
9761 language: &str,
9762 candidates: &[Candidate],
9763 base_short: &str,
9764 ) -> String {
9765 let mut text = format!(
9766 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9767 prompt::final_vote(labels, language),
9768 self.state.instruction,
9769 self.candidate_block(candidates, base_short)
9770 );
9771 if let Some(own) = self
9772 .state
9773 .judgements
9774 .get(j)
9775 .filter(|r| !r.ranking.is_empty())
9776 {
9777 let reasons = own
9778 .reasons
9779 .iter()
9780 .map(|(k, v)| format!("- {k}: {v}"))
9781 .collect::<Vec<_>>()
9782 .join("\n");
9783 text.push_str(&format!(
9784 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9785 own.ranking.iter().collect::<String>(),
9786 if reasons.is_empty() {
9787 ""
9788 } else {
9789 ", because:\n"
9790 }
9791 ));
9792 }
9793 if !self.state.deliberation.is_empty() {
9794 text.push_str("\n# What was argued before this vote\n");
9795 for t in self.transcript(&[], j) {
9796 text.push_str(&format!(
9797 "\n## {}{}\n\n{}\n",
9798 t.who,
9799 if t.is_self { " (you)" } else { "" },
9800 t.body.trim()
9801 ));
9802 }
9803 }
9804 text
9805 }
9806
9807 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9814 let mut turns = Vec::new();
9815 for j in &self.state.judgements {
9816 if j.ranking.is_empty() {
9817 continue;
9818 }
9819 let reasons = j
9820 .reasons
9821 .iter()
9822 .map(|(k, v)| format!("- {k}: {v}"))
9823 .collect::<Vec<_>>()
9824 .join("\n");
9825 turns.push(Turn {
9826 who: format!("Judge {} (opening ranking)", j.judge),
9827 is_self: j.judge == self_idx + 1,
9828 body: format!(
9829 "Ranked {}{}{reasons}",
9830 j.ranking.iter().collect::<String>(),
9831 if reasons.is_empty() {
9832 ""
9833 } else {
9834 ", because:\n"
9835 }
9836 ),
9837 });
9838 }
9839 for t in self
9840 .state
9841 .deliberation
9842 .iter()
9843 .flat_map(|r| r.turns.iter())
9844 .chain(current)
9845 {
9846 turns.push(Turn {
9847 who: format!("Judge {}", t.judge),
9848 is_self: t.judge == self_idx + 1,
9849 body: t.body.clone(),
9850 });
9851 }
9852 turns
9853 }
9854}
9855
9856fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9858 agent::has_session(spec.kind, seat, sessions)
9859}
9860
9861fn next_untried_in_roster<'a>(
9882 roster: &'a [AgentSpec],
9883 start: usize,
9884 tried: &BTreeSet<String>,
9885) -> Option<&'a AgentSpec> {
9886 roster
9887 .get(start + 1..)?
9888 .iter()
9889 .find(|s| !tried.contains(&s.id))
9890}
9891
9892fn pick_successor<'a>(
9913 roster: &'a [AgentSpec],
9914 start: usize,
9915 tried: &BTreeSet<String>,
9916 carried: Option<&BTreeSet<String>>,
9917 others: &BTreeSet<String>,
9918) -> Option<&'a AgentSpec> {
9919 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9920 roster
9921 .get(start + 1..)
9922 .and_then(|tail| {
9923 tail.iter()
9924 .filter(free)
9925 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9926 })
9927 .or_else(|| {
9928 carried
9929 .is_some()
9930 .then(|| roster.iter().find(free))
9931 .flatten()
9932 })
9933 .or_else(|| match carried {
9934 Some(c) => next_for_seat(roster, start, tried, c),
9935 None => next_untried_in_roster(roster, start, tried),
9936 })
9937}
9938
9939fn next_for_seat<'a>(
9949 roster: &'a [AgentSpec],
9950 start: usize,
9951 round_tried: &BTreeSet<String>,
9952 carried_failed: &BTreeSet<String>,
9953) -> Option<&'a AgentSpec> {
9954 roster
9955 .get(start + 1..)?
9956 .iter()
9957 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9958 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9959}
9960
9961fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9967 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9968 return spec;
9969 };
9970 let ok = |id: &str| !h.failed.contains(id);
9971 if let Some(last) = h.last_ok.as_deref()
9972 && ok(last)
9973 && let Some(s) = roster.iter().find(|s| s.id == last)
9974 {
9975 return s.clone();
9976 }
9977 if ok(&spec.id) {
9978 return spec;
9979 }
9980 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9981 roster
9982 .get(from + 1..)
9983 .into_iter()
9984 .flatten()
9985 .chain(roster.iter())
9986 .find(|s| ok(&s.id))
9987 .cloned()
9988 .unwrap_or(spec)
9989}
9990
9991pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9995 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9996}
9997
9998const TIMED_OUT: &str = "timed out";
10001
10002impl FailClass {
10003 fn of(out: &AgentOutcome) -> Option<Self> {
10005 match out {
10006 AgentOutcome::Ok(_) => None,
10007 AgentOutcome::Quota(_) => Some(Self::Quota),
10008 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10009 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10010 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10011 AgentOutcome::ResourceBlocked(_) => None,
10014 }
10015 }
10016
10017 fn stem_word(&self) -> &'static str {
10019 match self {
10020 Self::Quota => "quota",
10021 _ => "handover",
10022 }
10023 }
10024}
10025
10026fn failure_signature(msg: &str) -> String {
10030 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10031 if let Some(code) = line
10034 .strip_prefix("process exited with no usable output (exit code ")
10035 .and_then(|s| s.strip_suffix(')'))
10036 .and_then(|s| s.parse::<i32>().ok())
10037 {
10038 line = format!("exited with some({code}) and no usable output");
10039 } else if matches!(
10040 line.as_str(),
10041 "process exited with no usable output (no exit code reported)"
10042 | "process exited with no usable output (exit code unavailable)"
10043 ) {
10044 line = "exited with none and no usable output".to_owned();
10045 }
10046 let mut out = Vec::new();
10047 for word in line.split_whitespace() {
10048 if word.contains('/') || word.contains('\\') {
10049 out.push("<path>".to_owned());
10050 continue;
10051 }
10052 let mut w = String::new();
10053 let mut in_digits = false;
10054 for c in word.chars() {
10055 if c.is_ascii_digit() {
10056 if !in_digits {
10057 w.push('#');
10058 }
10059 in_digits = true;
10060 } else {
10061 in_digits = false;
10062 w.push(c);
10063 }
10064 }
10065 out.push(w);
10066 }
10067 out.join(" ").chars().take(120).collect()
10068}
10069
10070fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10077 match cur {
10078 FailClass::Quota | FailClass::Timeout => true,
10079 FailClass::Other(_) => prev != Some(cur),
10080 }
10081}
10082
10083fn fail_reason(out: &AgentOutcome) -> String {
10085 match out {
10086 AgentOutcome::Ok(_) => String::new(),
10087 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10088 AgentOutcome::Dropped(o) => format!(
10089 "the CLI dropped the stream ({})",
10090 o.dropped
10091 .as_ref()
10092 .map(|d| d.why.as_str())
10093 .unwrap_or("it ended without delivering its answer")
10094 ),
10095 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10096 e.lines().next().unwrap_or("").chars().take(160).collect()
10097 }
10098 }
10099}
10100
10101pub(crate) fn record_handover(
10104 state: &mut RunState,
10105 node: &str,
10106 seat: &str,
10107 from: &str,
10108 to: &str,
10109 class: &FailClass,
10110 reason: &str,
10111) {
10112 let message = if *class == FailClass::Quota {
10113 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10114 } else {
10115 format!("{seat}: handed over {from} -> {to} ({reason})")
10116 };
10117 state.event(node, message);
10118 state.handovers.push(Handover {
10119 at: Timestamp::now(),
10120 node: node.to_owned(),
10121 seat: seat.to_owned(),
10122 from: from.to_owned(),
10123 to: to.to_owned(),
10124 reason: reason.to_owned(),
10125 });
10126}
10127
10128fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10142 commands.iter().any(|c| c.exit_code.is_none())
10143}
10144
10145fn verified_noop_claim(
10158 usable: bool,
10159 commands: &[agent::CommandEvidence],
10160 text: &str,
10161) -> Option<String> {
10162 (usable && !has_unconfirmed_command(commands))
10163 .then(|| verdict::verified_noop(text))
10164 .flatten()
10165}
10166
10167fn short(commit: &str) -> String {
10168 commit.chars().take(7).collect()
10169}
10170
10171fn make_executable(path: &Path) -> Result<()> {
10172 #[cfg(unix)]
10173 {
10174 use std::os::unix::fs::PermissionsExt as _;
10175 let mut perms = std::fs::metadata(path)?.permissions();
10176 perms.set_mode(0o755);
10177 std::fs::set_permissions(path, perms)?;
10178 }
10179 #[cfg(not(unix))]
10180 {
10181 let _ = path;
10182 }
10183 Ok(())
10184}
10185
10186struct WaveCtx<'a> {
10193 run: &'a str,
10196 node: &'a str,
10198 prompts: &'a Prompts,
10199 cache: Option<&'a Path>,
10201 round: Option<usize>,
10204 carry_seats: bool,
10208 pauses: Option<(&'a Pause, &'a Pause)>,
10211}
10212
10213fn repository_wait(
10215 state: &RunState,
10216 pauses: Option<(&Pause, &Pause)>,
10217) -> crate::repo_lock::WaitPolicy {
10218 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10219 let inherited = crate::repo_lock::WaitPolicy::current();
10220 crate::repo_lock::WaitPolicy::new(
10221 Duration::from_secs(state.config.graph.timeout_resource_wait),
10222 move || {
10223 inherited.cancelled()
10224 || pauses
10225 .as_ref()
10226 .is_some_and(|(a, b)| a.parked() || b.parked())
10227 },
10228 )
10229}
10230
10231fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10232 if error.is::<crate::repo_lock::ResourceBlocked>() {
10233 state.resource_block = Some(crate::run::ResourceBlock {
10234 cause: crate::run::ResourceBlockCause::Other,
10235 node: "repository".into(),
10236 kind: if error.to_string().contains("disk pressure") {
10237 "disk pressure"
10238 } else {
10239 "repository coordination"
10240 }
10241 .into(),
10242 detail: error.to_string(),
10243 at: Timestamp::now(),
10244 });
10245 state.event("resource", error.to_string());
10246 }
10247}
10248
10249async fn coordinate_operation(
10250 state: &mut RunState,
10251 cwd: &Path,
10252 wait: &crate::repo_lock::WaitPolicy,
10253) -> Result<Option<crate::repo_lock::SeatLease>> {
10254 let result =
10255 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10256 .await;
10257 if let Err(e) = &result {
10258 record_repository_block(state, e);
10259 }
10260 result
10261}
10262
10263async fn run_one(
10265 job: SeatJob,
10266 sem: Arc<Semaphore>,
10267 ctx: &WaveCtx<'_>,
10268 state: &mut RunState,
10269 attempt: usize,
10270) -> (SeatState, AgentOutcome) {
10271 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10272 .await
10273 .pop()
10274 .expect("one job in, one result out");
10275 (seat, out)
10276}
10277
10278async fn wave(
10284 jobs: Vec<SeatJob>,
10285 sem: Arc<Semaphore>,
10286 ctx: &WaveCtx<'_>,
10287 state: &mut RunState,
10288 attempt: usize,
10289) -> Vec<(usize, SeatState, AgentOutcome)> {
10290 let WaveCtx {
10291 run,
10292 node,
10293 prompts,
10294 cache,
10295 round,
10296 carry_seats: _,
10297 pauses,
10298 } = *ctx;
10299 for job in &jobs {
10300 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10301 }
10302 if let Err(e) = state.save() {
10303 tracing::warn!("could not persist in-progress seats: {e:#}");
10308 }
10309 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10325 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10329 let cache_guard = if let Some(cache_dir) = cache {
10330 if jobs_had_a_writer {
10331 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10332 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10333 match acquire_cache_lease(
10334 state,
10335 cache_dir,
10336 &owner,
10337 resource_limit,
10338 node,
10339 node,
10340 &cancel,
10341 )
10342 .await
10343 {
10344 Ok(Some(g)) => Some(g),
10345 Ok(None) => {
10348 if let Some(block) = &mut state.resource_block {
10349 block.cause = crate::run::ResourceBlockCause::Cancelled {
10350 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10351 };
10352 }
10353 return block_wave(
10354 jobs,
10355 state,
10356 node,
10357 attempt,
10358 "parked while waiting for the shared build cache",
10359 );
10360 }
10361 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10364 Err(e) => {
10365 let why = format!("{e:#}");
10369 state.event(
10370 "resource",
10371 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10372 );
10373 return block_wave(jobs, state, node, attempt, &why);
10374 }
10375 }
10376 } else {
10377 None
10378 }
10379 } else {
10380 None
10381 };
10382 state.seats_spawning(node, attempt);
10385 if let Err(e) = state.save() {
10386 tracing::warn!("could not persist the start of a wave: {e:#}");
10387 }
10388 let slot_limit = resource_limit
10392 + jobs
10393 .iter()
10394 .map(|j| j.timeout)
10395 .max()
10396 .unwrap_or_default()
10397 .saturating_mul(jobs.len() as u32);
10398 let mut set = tokio::task::JoinSet::new();
10399 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10402 let overlay = prompts.overlay(node);
10403 let repository_wait = repository_wait(state, pauses);
10404 for (i, mut job) in jobs.into_iter().enumerate() {
10405 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10406 if cache.is_some() {
10407 job.prompt.push('\n');
10408 job.prompt
10409 .push_str(&prompt::build_cache_note(node, job.allow_write));
10410 }
10411 let sem = Arc::clone(&sem);
10412 let run = run.to_owned();
10413 let node = node.to_owned();
10414 let attachments = if node == "implement" {
10417 state.attachments.clone()
10418 } else {
10419 Vec::new()
10420 };
10421 let cache = cache
10432 .filter(|_| job.allow_write && cache_guard.is_some())
10433 .map(Path::to_path_buf);
10434 let spawned_tx = spawned_tx.clone();
10435 let repository_wait = repository_wait.clone();
10436 let coordination = state.config.coordination.clone();
10437 let fetched_base = state.fetched_base.clone();
10438 set.spawn(async move {
10439 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10443 Ok(p) => p,
10444 Err(_) => {
10445 return (
10446 i,
10447 job.seat,
10448 AgentOutcome::ResourceBlocked(format!(
10449 "no parallel slot became free within {}s",
10450 slot_limit.as_secs()
10451 )),
10452 false,
10453 );
10454 }
10455 };
10456 spawned_tx.send(job.seat.key.clone()).ok();
10457 let mut seat = job.seat;
10458 let out = repository_wait
10459 .scope(agent::invoke(
10460 &job.spec,
10461 &mut seat,
10462 &Invocation {
10463 coordination: coordination.as_ref(),
10464 fetched_base: fetched_base.as_ref(),
10465 cwd: &job.cwd,
10466 prompt: &job.prompt,
10467 timeout: job.timeout,
10468 allow_write: job.allow_write,
10469 sessions: job.sessions,
10470 artifacts: &job.artifacts,
10471 stem: &job.stem,
10472 run: &run,
10473 node: &node,
10474 cache_dir: cache.as_deref(),
10475 attachments: &attachments,
10476 writable: &[],
10477 },
10478 ))
10479 .await;
10480 let interrupted = out
10481 .as_ref()
10482 .err()
10483 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10484 let out = match out {
10485 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10486 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10487 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10495 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10496 Ok(o) => AgentOutcome::Failed(format!(
10497 "process exited with no usable output ({})",
10498 crate::notices::exit_status(o.exit_code)
10499 )),
10500 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10501 AgentOutcome::ResourceBlocked(e.to_string())
10502 }
10503 Err(e) => AgentOutcome::Failed(e.to_string()),
10504 };
10505 (i, seat, out, interrupted)
10506 });
10507 }
10508 drop(spawned_tx);
10509 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10510 loop {
10511 let joined = tokio::select! {
10512 Some(key) = spawned_rx.recv() => {
10513 state.seat_spawned(&key);
10515 if let Err(e) = state.save() {
10516 tracing::warn!("could not persist a seat's start: {e:#}");
10517 }
10518 continue;
10519 }
10520 joined = set.join_next() => joined,
10521 };
10522 let Some(joined) = joined else {
10523 break;
10524 };
10525 let (i, seat, out, interrupted) = match joined {
10526 Ok(v) => v,
10527 Err(e) => {
10531 tracing::error!("agent task panicked: {e}");
10532 continue;
10533 }
10534 };
10535 state.seat_finished(&seat.key);
10536 if interrupted
10537 && node == "implement"
10538 && let Some(label) = state
10539 .candidates
10540 .iter()
10541 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10542 .map(|candidate| candidate.label)
10543 {
10544 state.candidate_preservation.insert(
10545 label,
10546 "interrupted implementation awaiting persistence".into(),
10547 );
10548 state.interrupted_implementations.insert(label);
10549 }
10550 if let AgentOutcome::ResourceBlocked(why) = &out
10551 && state.resource_block.is_none()
10552 {
10553 state.resource_block = Some(crate::run::ResourceBlock {
10554 cause: crate::run::ResourceBlockCause::Other,
10555 node: node.to_owned(),
10556 kind: if why.contains("disk pressure") {
10557 "disk pressure"
10558 } else if why.contains("repository coordination") {
10559 "repository coordination"
10560 } else {
10561 "parallel slot"
10562 }
10563 .to_owned(),
10564 detail: why.clone(),
10565 at: Timestamp::now(),
10566 });
10567 let activity = if interrupted {
10568 "agent execution interrupted"
10569 } else {
10570 "no agent was started"
10571 };
10572 state.event("resource", format!("{node}: {activity} - {why}"));
10573 }
10574 record_jobs(state, node, round, &seat.key, &out);
10575 if let Err(e) = state.save() {
10576 tracing::warn!("could not persist a seat's completion: {e:#}");
10577 }
10578 if collected.len() <= i {
10579 collected.resize_with(i + 1, || None);
10580 }
10581 collected[i] = Some((i, seat, out));
10582 }
10583 if state
10589 .active
10590 .values()
10591 .any(|a| a.node == node && a.attempt == attempt)
10592 {
10593 state
10594 .active
10595 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10596 if let Err(e) = state.save() {
10597 tracing::warn!("could not persist the end of a wave: {e:#}");
10598 }
10599 }
10600 if let Some(cache_dir) = cache
10607 && jobs_had_a_writer
10608 {
10609 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10610 }
10611 if let Some(guard) = cache_guard {
10612 guard.release();
10613 }
10614 collected.into_iter().flatten().collect()
10615}
10616
10617fn record_jobs(
10628 state: &mut RunState,
10629 node: &str,
10630 round: Option<usize>,
10631 seat: &str,
10632 out: &AgentOutcome,
10633) {
10634 let commands: &[agent::CommandEvidence] = match out {
10635 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10636 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10637 };
10638 let checked_at = Timestamp::now();
10639 for c in commands {
10640 state.jobs.push(JobRecord {
10641 node: node.to_owned(),
10642 round,
10643 seat: seat.to_owned(),
10644 id: c.id.clone(),
10645 description: c.description.clone(),
10646 checked_at,
10647 status: match c.exit_code {
10648 Some(0) => JobStatus::Completed,
10649 Some(_) => JobStatus::Failed,
10650 None => JobStatus::Unknown,
10651 },
10652 exit_code: c.exit_code,
10653 result_summary: c.result_summary.clone(),
10654 source: c.source.clone(),
10655 });
10656 }
10657}
10658
10659fn round_is_clean(
10680 blocking: usize,
10681 e2e_ok: bool,
10682 answered: usize,
10683 expected: usize,
10684 quota_missing: usize,
10685 policy: IncompleteReviewPolicy,
10686) -> bool {
10687 if blocking != 0 || !e2e_ok {
10688 return false;
10689 }
10690 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10691 return true;
10692 }
10693 answered > 0 && expected - answered <= quota_missing
10694}
10695
10696pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10720 if max_rounds == 0
10721 || reviews.last().is_some_and(|r| {
10722 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10723 })
10724 {
10725 return Some(RunStatus::Gating);
10726 }
10727 let last = reviews.last()?;
10728 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10729 if reviews.len() < max_rounds && !stagnant {
10730 return None;
10731 }
10732 if last.incomplete() && last.blocking == 0 {
10733 return Some(RunStatus::Blocked);
10734 }
10735 if last.e2e_status() == E2eStatus::ResourceBlocked
10736 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10737 {
10738 return None;
10739 }
10740 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10741 RunStatus::Gating
10742 } else {
10743 RunStatus::Blocked
10744 })
10745}
10746
10747pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10762 if nudged {
10763 (full / 4).max(Duration::from_secs(120)).min(full)
10764 } else {
10765 full
10766 }
10767}
10768
10769#[allow(clippy::too_many_arguments)]
10798async fn ask_json_wave<T>(
10799 jobs: Vec<SeatJob>,
10800 sem: Arc<Semaphore>,
10801 retries: usize,
10802 roster: &[AgentSpec],
10803 ctx: &WaveCtx<'_>,
10804 losses: &mut Vec<QuotaLoss>,
10805 state: &mut RunState,
10806 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10807) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10808where
10809 T: serde::de::DeserializeOwned + Send + 'static,
10810{
10811 ask_wave_with(
10812 jobs,
10813 sem,
10814 retries,
10815 roster,
10816 ctx,
10817 losses,
10818 state,
10819 &|text: &str| {
10820 let v = verdict::extract_json::<T>(text)?;
10821 validate(&v)?;
10822 Ok(v)
10823 },
10824 )
10825 .await
10826}
10827
10828#[allow(clippy::too_many_arguments)]
10837async fn ask_wave_with<T>(
10838 jobs: Vec<SeatJob>,
10839 sem: Arc<Semaphore>,
10840 retries: usize,
10841 roster: &[AgentSpec],
10842 ctx: &WaveCtx<'_>,
10843 losses: &mut Vec<QuotaLoss>,
10844 state: &mut RunState,
10845 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10846) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10847where
10848 T: Send + 'static,
10849{
10850 let n = jobs.len();
10851 let originals: Vec<SeatJob> = jobs;
10852 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10853 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10854 let mut nudges: Vec<usize> = vec![0; n];
10861 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10865 let mut tried: Vec<BTreeSet<String>> = specs
10866 .iter()
10867 .map(|s| BTreeSet::from([s.id.clone()]))
10868 .collect();
10869 let starts: Vec<usize> = specs
10870 .iter()
10871 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10872 .collect();
10873 let carry = ctx.carry_seats && !roster.is_empty();
10874 let carried: Vec<BTreeSet<String>> = originals
10877 .iter()
10878 .map(|j| {
10879 state
10880 .seat_history
10881 .get(&j.seat.key)
10882 .filter(|_| carry)
10883 .map(|h| h.failed.clone())
10884 .unwrap_or_default()
10885 })
10886 .collect();
10887 let mut prev: Vec<Option<FailClass>> = originals
10888 .iter()
10889 .map(|j| {
10890 state
10891 .seat_history
10892 .get(&j.seat.key)
10893 .filter(|_| carry)
10894 .and_then(|h| h.last_fail.clone())
10895 })
10896 .collect();
10897 let next_agent =
10898 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10899 let others: BTreeSet<String> = specs
10900 .iter()
10901 .enumerate()
10902 .filter(|(j, _)| *j != i)
10903 .map(|(_, s)| s.id.clone())
10904 .collect();
10905 pick_successor(
10906 roster,
10907 starts[i],
10908 tried,
10909 carry.then(|| &carried[i]),
10910 &others,
10911 )
10912 .cloned()
10913 };
10914 let mut fresh: Vec<Option<String>> = vec![None; n];
10915 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
10916 let mut pending: Vec<usize> = (0..n).collect();
10917
10918 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10923 for round in 0..max_rounds {
10924 if pending.is_empty() {
10925 break;
10926 }
10927 let mut batch = Vec::with_capacity(pending.len());
10928 let mut renudged: Vec<&str> = Vec::new();
10929 for &i in &pending {
10930 let src = &originals[i];
10931 let full: &str = match &src.handover {
10936 Some(h) if specs[i].id != src.spec.id => h,
10937 _ => &src.prompt,
10938 };
10939 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10942 (
10943 full.to_owned(),
10944 src.timeout,
10945 format!("{}-{word}-{}", src.stem, specs[i].id),
10946 )
10947 } else if nudges[i] == 0 {
10948 (full.to_owned(), src.timeout, src.stem.clone())
10949 } else {
10950 renudged.push(src.seat.key.as_str());
10951 let why = done[i]
10952 .as_ref()
10953 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10954 .unwrap_or_else(|| "no parsable answer".to_owned());
10955 let nudge = prompt::nudge(&why);
10956 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10957 let prompt = if nudged {
10958 nudge
10959 } else {
10960 format!("{full}\n\n---\n\n{nudge}")
10961 };
10962 (
10963 prompt,
10964 retry_budget(src.timeout, nudged),
10965 format!("{}-retry{}", src.stem, nudges[i]),
10966 )
10967 };
10968 batch.push(SeatJob {
10969 spec: specs[i].clone(),
10970 seat: seats[i].clone(),
10971 cwd: src.cwd.clone(),
10972 prompt,
10973 timeout,
10974 allow_write: src.allow_write,
10975 sessions: src.sessions,
10976 artifacts: src.artifacts.clone(),
10977 stem,
10978 handover: None,
10979 });
10980 }
10981
10982 if !renudged.is_empty() {
10983 state.event(
10984 ctx.node,
10985 format!("retry {round}: re-asking {}", renudged.join(", ")),
10986 );
10987 }
10988 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10989 let mut still = Vec::new();
10990 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10991 seats[i] = seat;
10992 if let AgentOutcome::ResourceBlocked(why) = &out {
10995 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10996 continue;
10997 }
10998 let class = FailClass::of(&out);
10999 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11002 && nudges[i] < retries
11003 && !roster.is_empty();
11004 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11005 let next = should_hand_over(prev[i].as_ref(), &cur)
11006 .then(|| next_agent(i, &tried[i], &specs))
11007 .flatten();
11008 if carry {
11009 let h = state
11010 .seat_history
11011 .entry(originals[i].seat.key.clone())
11012 .or_default();
11013 h.failed.insert(specs[i].id.clone());
11014 h.last_fail = Some(cur.clone());
11015 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11016 h.last_ok = None;
11017 }
11018 if let Err(e) = state.save() {
11021 tracing::warn!("could not persist a seat's failure history: {e:#}");
11022 }
11023 }
11024 if let Some(next) = next {
11025 record_handover(
11026 state,
11027 ctx.node,
11028 &originals[i].seat.key,
11029 &specs[i].id,
11030 &next.id,
11031 &cur,
11032 &fail_reason(&out),
11033 );
11034 tried[i].insert(next.id.clone());
11035 prev[i] = Some(cur.clone());
11036 seats[i] =
11037 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11038 specs[i] = next;
11039 fresh[i] = Some(cur.stem_word().to_owned());
11040 nudges[i] = 0;
11041 done[i] = Some(Err(anyhow::anyhow!(
11042 "handed over after: {}",
11043 fail_reason(&out)
11044 )));
11045 still.push(i);
11046 continue;
11047 }
11048 }
11049 if carry
11050 && !nudge_first
11051 && let Some(cur) = class.clone()
11052 {
11053 let h = state
11055 .seat_history
11056 .entry(originals[i].seat.key.clone())
11057 .or_default();
11058 h.failed.insert(specs[i].id.clone());
11059 h.last_fail = Some(cur);
11060 }
11061 let parsed = match out {
11062 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11063 AgentOutcome::Quota(o) => {
11064 last_quota[i] = o.quota.clone();
11065 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11066 }
11067 AgentOutcome::Dropped(o) => {
11071 let why = o
11072 .dropped
11073 .as_ref()
11074 .map(|d| d.why.as_str())
11075 .unwrap_or("the CLI ended the stream without delivering its answer");
11076 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11077 }
11078 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11079 Err(anyhow::anyhow!(e))
11080 }
11081 };
11082 let quota = class == Some(FailClass::Quota);
11083 let failed = parsed.is_err();
11084 if !failed && matches!(ctx.node, "judge" | "vote") {
11085 state
11088 .quota
11089 .retain(|loss| loss.seat != originals[i].seat.key);
11090 }
11091 done[i] = Some(parsed);
11092 if carry && !failed {
11093 let h = state
11094 .seat_history
11095 .entry(originals[i].seat.key.clone())
11096 .or_default();
11097 h.failed.remove(&specs[i].id);
11098 h.last_ok = Some(specs[i].id.clone());
11099 h.last_fail = None;
11100 }
11101 let agent_failure = class.is_some()
11109 && !nudge_first
11110 && !roster.is_empty()
11111 && next_agent(i, &tried[i], &specs).is_some();
11112 if failed && !quota && !agent_failure && nudges[i] < retries {
11113 nudges[i] += 1;
11114 still.push(i);
11115 }
11116 }
11117 pending = still;
11118 }
11119
11120 for (i, q) in last_quota.into_iter().enumerate() {
11123 if let Some(quota) = q {
11124 losses.push(QuotaLoss::from_report(
11125 originals[i].seat.key.clone(),
11126 ctx.node,
11127 Some("a),
11128 ));
11129 }
11130 }
11131
11132 if state.status != RunStatus::Stalled
11135 && ctx.node != "deliberate"
11136 && n > 0
11137 && losses.len() == n
11138 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11139 {
11140 block_machine_outage(state, ctx.node, losses);
11141 state.quota.append(losses);
11142 }
11143
11144 seats
11145 .into_iter()
11146 .zip(done)
11147 .zip(nudges)
11148 .map(|((seat, res), attempts)| {
11149 (
11150 seat,
11151 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11152 attempts,
11153 )
11154 })
11155 .collect()
11156}
11157
11158pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11161 let mut auth: Vec<_> = losses
11162 .iter()
11163 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11164 .map(|loss| loss.agent.clone())
11165 .collect();
11166 auth.sort();
11167 auth.dedup();
11168 let (kind, detail) = if auth.is_empty() {
11169 (
11170 "agent usage limit",
11171 "All seats reached their agent usage limits; resume after usage is available"
11172 .to_owned(),
11173 )
11174 } else {
11175 (
11176 "agent authentication",
11177 format!(
11178 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11179 auth.join(", ")
11180 ),
11181 )
11182 };
11183 state.resource_block = Some(crate::run::ResourceBlock {
11184 node: node.to_owned(),
11185 kind: kind.to_owned(),
11186 cause: crate::run::ResourceBlockCause::Other,
11187 detail: detail.clone(),
11188 at: Timestamp::now(),
11189 });
11190 state.event(node, detail);
11191}
11192
11193fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11195 results
11196 .iter()
11197 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11198}
11199
11200fn block_wave(
11204 jobs: Vec<SeatJob>,
11205 state: &mut RunState,
11206 node: &str,
11207 attempt: usize,
11208 why: &str,
11209) -> Vec<(usize, SeatState, AgentOutcome)> {
11210 state.resource_wait = None;
11211 state
11212 .active
11213 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11214 if let Err(e) = state.save() {
11215 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11216 }
11217 jobs.into_iter()
11218 .enumerate()
11219 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11220 .collect()
11221}
11222
11223async fn acquire_cache_lease(
11236 state: &mut RunState,
11237 cache_dir: &Path,
11238 owner: &crate::cache::Owner,
11239 budget: Duration,
11240 node: &str,
11241 context: &str,
11242 cancel: &(dyn Fn() -> bool + Sync),
11243) -> Result<Option<crate::cache::Guard>> {
11244 let home = crate::run::home();
11245 let started = Instant::now();
11246 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11247 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11248 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11249 Err(e) => {
11250 state.event(
11251 "verify",
11252 format!("{context}: could not check the shared build cache: {e:#}"),
11253 );
11254 if let Err(e2) = state.save() {
11255 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11256 }
11257 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11258 }
11259 };
11260 state.event(
11261 "verify",
11262 format!(
11263 "{context}: waiting for the shared build cache at {} ({})",
11264 cache_dir.display(),
11265 busy.describe()
11266 ),
11267 );
11268 state.resource_wait = Some(crate::run::ResourceWait {
11271 node: context.to_owned(),
11272 kind: "shared build cache".to_owned(),
11273 holder: busy.describe(),
11274 since: Timestamp::now(),
11275 limit_secs: budget.as_secs(),
11276 });
11277 if let Err(e) = state.save() {
11278 tracing::warn!("could not persist a cache wait: {e:#}");
11279 }
11280 let remaining = budget.saturating_sub(started.elapsed());
11281 let waited = crate::cache::wait_for_unless(
11282 &home,
11283 cache_dir,
11284 owner,
11285 remaining,
11286 Duration::from_secs(5),
11287 cancel,
11288 )
11289 .await;
11290 state.resource_wait = None;
11291 let cause = match &waited {
11292 Ok(Some(_)) => None,
11293 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11294 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11295 Some(crate::run::ResourceBlockCause::Timeout {
11296 waited_secs: started.elapsed().as_secs(),
11297 limit_secs: budget.as_secs(),
11298 })
11299 }
11300 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11301 };
11302 if let Some(cause) = cause {
11303 let detail = match &waited {
11304 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11305 Err(error) => format!("{error:#}"),
11306 };
11307 state.resource_block = Some(crate::run::ResourceBlock {
11308 cause,
11309 node: node.to_owned(),
11310 kind: "shared build cache".into(),
11311 detail,
11312 at: Timestamp::now(),
11313 });
11314 state.event(
11315 "resource",
11316 state.resource_block.as_ref().unwrap().description(),
11317 );
11318 }
11319 if let Err(error) = state.save() {
11320 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11321 }
11322 waited
11323}
11324
11325#[derive(Debug)]
11330struct CacheCheckError(String);
11331
11332impl std::fmt::Display for CacheCheckError {
11333 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11334 f.write_str(&self.0)
11335 }
11336}
11337
11338impl std::error::Error for CacheCheckError {}
11339
11340fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11342 state
11343 .resource_block
11344 .get_or_insert_with(|| crate::run::ResourceBlock {
11345 cause: crate::run::ResourceBlockCause::Other,
11346 node: node.to_owned(),
11347 kind: "shared build cache".into(),
11348 detail: format!("{error:#}"),
11349 at: Timestamp::now(),
11350 });
11351 if let Err(error) = state.save() {
11352 tracing::warn!("could not persist cache resource block: {error:#}");
11353 }
11354}
11355
11356#[allow(clippy::too_many_arguments)]
11377async fn with_cache_lease<'s, F, Fut>(
11378 state: &'s mut RunState,
11379 cache_dir: Option<&Path>,
11380 node: &str,
11381 seat: &str,
11382 worktree: &Path,
11383 head: &str,
11384 budget: Duration,
11385 context: &str,
11386 wait: &crate::repo_lock::WaitPolicy,
11387 body: F,
11388) -> Result<(Vec<CommandOutcome>, bool)>
11389where
11390 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11391 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11392{
11393 let started = Instant::now();
11396 let home = crate::run::home();
11397 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11398 let guard = if let Some(cache_dir) = cache_dir {
11399 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11400 match acquire_cache_lease(
11401 state,
11402 cache_dir,
11403 &owner,
11404 resource_limit,
11405 node,
11406 context,
11407 &|| wait.cancelled(),
11408 )
11409 .await
11410 .and_then(|g| {
11411 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11412 }) {
11413 Ok(g) => Some(g),
11414 Err(e) => {
11415 record_cache_error(state, node, &e);
11416 return Ok((
11417 vec![CommandOutcome {
11418 command: "(waiting for the shared build cache)".into(),
11419 code: None,
11420 output_tail: e.to_string(),
11421 duration_ms: started.elapsed().as_millis() as u64,
11422 resource_blocked: true,
11423 }],
11424 false,
11425 ));
11426 }
11427 }
11428 } else {
11429 None
11430 };
11431 let _coordination = coordinate_operation(state, worktree, wait).await?;
11432 if let Some(cache_dir) = cache_dir {
11433 let identity = crate::cache::Identity::new(worktree, head);
11434 if let Err(e) = crate::cache::ensure_fresh(
11435 &home,
11436 cache_dir,
11437 &identity,
11438 &state.config.verify.cargo_manifests(),
11439 ) {
11440 record_cache_error(state, node, &e);
11441 state.event(
11442 "verify",
11443 format!(
11444 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11445 worktree.display(),
11446 short(head)
11447 ),
11448 );
11449 if let Some(guard) = guard {
11450 guard.release();
11451 }
11452 return Ok((
11453 vec![CommandOutcome {
11454 command: "(confirming the shared build cache is fresh)".into(),
11455 code: None,
11456 output_tail: e.to_string(),
11457 duration_ms: started.elapsed().as_millis() as u64,
11458 resource_blocked: true,
11459 }],
11460 false,
11461 ));
11462 }
11463 }
11464 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11465 if !timed_out_pids.is_empty() {
11466 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11467 }
11468 drop(_coordination);
11471 if let Some(guard) = guard {
11472 guard.release();
11473 }
11474 Ok((outcomes, retried))
11475}
11476
11477async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11489 wait_for_pids_with(
11490 pids,
11491 crate::proc::pid_alive,
11492 LEASE_RELEASE_POLL,
11493 LEASE_RELEASE_MAX_WAIT,
11494 )
11495 .await;
11496}
11497
11498async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11504 pids: &[u32],
11505 alive: F,
11506 poll: Duration,
11507 max_wait: Duration,
11508) {
11509 let deadline = Instant::now() + max_wait;
11510 loop {
11511 if pids.iter().all(|&pid| !alive(pid)) {
11512 return;
11513 }
11514 if Instant::now() >= deadline {
11515 return;
11516 }
11517 tokio::time::sleep(poll).await;
11518 }
11519}
11520
11521fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11528 outcomes.iter().any(|o| o.resource_blocked)
11529}
11530
11531pub(crate) enum GateFix {
11533 Retry,
11535 Stop,
11538 Defer,
11541}
11542
11543pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11551 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11552 red.peek().is_some()
11553 && red.all(|o| {
11554 !o.resource_blocked
11555 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11556 && !o.output_tail.trim().is_empty()
11557 })
11558}
11559
11560fn e2e_outcome_label(o: &CommandOutcome) -> String {
11564 if o.ok() {
11565 return "pass".to_owned();
11566 }
11567 let reason = if o.build_failed() {
11568 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11569 } else {
11570 format!("FAIL ({:?})", o.code)
11571 };
11572 format!(
11573 "{reason}\n{}",
11574 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11575 )
11576}
11577
11578async fn run_e2e_with_retry(
11586 state: &mut RunState,
11587 shell: &[String],
11588 commands: &[String],
11589 worktree: &Path,
11590 timeout: Duration,
11591 context: &str,
11592) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11593 let (mut e2e, mut timed_out_pids) = run_commands(
11594 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11595 )
11596 .await;
11597 for o in &e2e {
11598 state.event(
11599 "verify",
11600 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11601 );
11602 }
11603 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11607 if verify_retried {
11608 state.event(
11609 "verify",
11610 format!(
11611 "{context}: verify could not build/link, not a test result — retrying once \
11612 before concluding"
11613 ),
11614 );
11615 let retried = run_commands(
11616 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11617 )
11618 .await;
11619 e2e = retried.0;
11620 timed_out_pids.extend(retried.1);
11623 for o in &e2e {
11624 state.event(
11625 "verify",
11626 format!(
11627 "{context}: retry `{}` -> {}",
11628 o.command,
11629 e2e_outcome_label(o)
11630 ),
11631 );
11632 }
11633 }
11634 (e2e, verify_retried, timed_out_pids)
11635}
11636
11637#[allow(clippy::too_many_arguments)]
11653async fn run_commands(
11654 state: &mut RunState,
11655 node: &str,
11656 task: &str,
11657 attempt: usize,
11658 shell: &[String],
11659 commands: &[String],
11660 cwd: &Path,
11661 timeout: Duration,
11662) -> (Vec<CommandOutcome>, Vec<u32>) {
11663 if commands.is_empty() {
11664 return (Vec::new(), Vec::new());
11669 }
11670 let mut out = Vec::new();
11671 let mut timed_out_pids = Vec::new();
11672 let total = commands.len();
11673 for (idx, command) in commands.iter().enumerate() {
11674 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11675 if let Err(e) = state.save() {
11676 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11677 }
11678 let started = Instant::now();
11679 let mut cmd = tokio::process::Command::new(&shell[0]);
11680 cmd.quiet();
11681 for key in crate::worker::INVOCATION_ENV {
11685 cmd.env_remove(key);
11686 }
11687 cmd.args(&shell[1..])
11688 .arg(command)
11689 .current_dir(cwd)
11690 .stdin(std::process::Stdio::null())
11691 .stdout(std::process::Stdio::piped())
11692 .stderr(std::process::Stdio::piped())
11693 .kill_on_drop(true);
11694 let mut outputs = Vec::new();
11695 let protected = (|| -> Result<()> {
11696 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11697 outputs.push(guard);
11698 }
11699 if let Some(cache) = state.config.cache_dir() {
11700 outputs.push(crate::retention::OutputLease::cache(
11701 &crate::run::home(),
11702 &cache,
11703 false,
11704 )?);
11705 }
11706 Ok(())
11707 })();
11708 if let Err(e) = protected {
11709 out.push(CommandOutcome {
11710 command: command.clone(),
11711 code: None,
11712 output_tail: format!("output ownership unavailable: {e:#}"),
11713 duration_ms: 0,
11714 resource_blocked: true,
11715 });
11716 continue;
11717 }
11718 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11719 let (code, body) = match spawned {
11720 Ok(mut running) => {
11721 let pid = running.child().id();
11722 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11723 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11724 let mut paths = vec![cwd.to_owned()];
11725 if let Some(cache) = state.config.cache_dir() {
11726 paths.push(cache);
11727 }
11728 let mut pressure = None;
11729 let status = tokio::select! {
11730 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11731 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11732 pressure = Some(reason); None
11733 }
11734 };
11735 let (code, failure) = match status {
11736 Some(Ok(Ok(status))) => (
11737 status.code(),
11738 status
11739 .code()
11740 .is_none()
11741 .then(|| format!("command terminated: {status}")),
11742 ),
11743 Some(Ok(Err(error))) => {
11744 (None, Some(format!("failed waiting for command: {error}")))
11745 }
11746 Some(Err(_)) => {
11747 if !running.coordinated()
11751 && let Some(pid) = pid
11752 {
11753 timed_out_pids.push(pid);
11754 }
11755 (
11756 None,
11757 Some(format!("timed out after {}s", timeout.as_secs())),
11758 )
11759 }
11760 None => (None, None),
11761 };
11762 running.finish().await;
11763 if let Some(reason) = pressure {
11764 state.resource_block = Some(crate::run::ResourceBlock {
11765 cause: crate::run::ResourceBlockCause::Other,
11766 node: node.to_owned(),
11767 kind: "disk pressure".to_owned(),
11768 detail: reason.clone(),
11769 at: Timestamp::now(),
11770 });
11771 out.push(CommandOutcome {
11772 command: command.clone(),
11773 code: None,
11774 output_tail: reason,
11775 duration_ms: started.elapsed().as_millis() as u64,
11776 resource_blocked: true,
11777 });
11778 break;
11779 }
11780 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11781 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11782 if let Some(failure) = failure {
11783 body.push('\n');
11784 body.push_str(&failure);
11785 }
11786 (code, body)
11787 }
11788 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11789 };
11790 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11791 if !body.is_empty() {
11792 match crate::run::write_verification_artifact(state, task, &body) {
11796 Ok(name) if summary != body => {
11797 summary = format!("[full output: artifacts/{name}]\n{summary}");
11798 }
11799 Ok(_) => {}
11800 Err(e) => {
11801 let warning = format!("could not save the full {task} output: {e:#}");
11802 tracing::warn!("{warning}");
11803 state.event(node, warning);
11804 if summary != body {
11805 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11806 }
11807 }
11808 }
11809 }
11810 out.push(CommandOutcome {
11811 command: command.clone(),
11812 code,
11813 output_tail: summary,
11814 duration_ms: started.elapsed().as_millis() as u64,
11815 resource_blocked: false,
11816 });
11817 }
11818 state.task_finished(task);
11819 if let Err(e) = state.save() {
11820 tracing::warn!("could not persist the end of {task}: {e:#}");
11821 }
11822 (out, timed_out_pids)
11823}
11824
11825fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11837 let repo = repo.display();
11838 match style {
11839 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11840 MergeStyle::Squash => {
11841 let subject = message
11844 .lines()
11845 .next()
11846 .unwrap_or(branch)
11847 .replace(['\\', '"', '$', '`'], "");
11848 format!(
11849 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11850 )
11851 }
11852 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11853 }
11854}
11855
11856const PR_TITLE_MAX: usize = 240;
11867
11868fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11879 let line = queue::first_line(text)?;
11880 let chars: Vec<char> = line.chars().collect();
11881 let (kept, mark) = if chars.len() <= max {
11882 (line.to_owned(), "")
11883 } else if let Some(end) = sentence_end(&chars, max) {
11884 (chars[..end].iter().collect::<String>(), "")
11885 } else {
11886 let room = max.saturating_sub(3);
11887 let cut = if chars[room].is_whitespace() {
11890 room
11891 } else {
11892 chars[..room]
11893 .iter()
11894 .rposition(|c| c.is_whitespace())
11895 .unwrap_or(room)
11896 };
11897 let head: String = chars[..cut].iter().collect();
11898 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11899 (head.to_owned(), "...")
11900 };
11901 if kept.is_empty() {
11902 return None;
11903 }
11904 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11905 return None;
11906 }
11907 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11908 return None;
11909 }
11910 Some(format!("{kept}{mark}"))
11911}
11912
11913fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11917 const MIN: usize = 20;
11918 for i in MIN..max.min(chars.len()) {
11919 if !matches!(chars[i], '.' | '!' | '?') {
11920 continue;
11921 }
11922 let Some(&next) = chars.get(i + 1) else {
11923 continue;
11924 };
11925 if !next.is_whitespace() {
11926 continue;
11927 }
11928 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11929 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11930 continue;
11931 }
11932 let word: String = chars[..i]
11933 .iter()
11934 .rev()
11935 .take_while(|c| !c.is_whitespace())
11936 .collect::<Vec<_>>()
11937 .into_iter()
11938 .rev()
11939 .collect();
11940 let word = word.to_ascii_lowercase();
11941 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11942 continue;
11943 }
11944 let end = chars[..i]
11945 .iter()
11946 .rposition(|c| !c.is_whitespace())
11947 .map_or(i, |p| p + 1);
11948 return Some(end);
11949 }
11950 None
11951}
11952
11953struct PrMessage {
11958 title: String,
11959 body: String,
11960}
11961
11962impl PrMessage {
11963 fn commit_message(&self) -> String {
11967 format!("{}\n\n{}", self.title, self.body)
11968 }
11969}
11970
11971fn title_marker(line: &str) -> Option<&str> {
11973 let line = line.trim();
11974 let head = line.get(..6)?;
11975 head.eq_ignore_ascii_case("title:")
11976 .then(|| line[6..].trim())
11977}
11978
11979fn summary_title(summary: &str) -> Option<String> {
11984 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11985 let raw = title_marker(first)?;
11986 if raw.is_empty() {
11987 return None;
11988 }
11989 let title = queue::title_from(raw, PR_TITLE_MAX);
11990 let lower = title.to_ascii_lowercase();
11991 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11992 return None;
11993 }
11994 Some(title)
11995}
11996
11997const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11999
12000pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12002
12003pub(crate) fn is_review_run(state: &RunState) -> bool {
12007 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12008}
12009
12010fn review_title(state: &RunState) -> Option<String> {
12015 english_subject(
12016 state.reviewed_commits.as_ref()?.first()?,
12017 state.config.graph.github_english(),
12018 )
12019}
12020
12021fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12024 let raw = raw.trim();
12025 if raw.is_empty() {
12026 return None;
12027 }
12028 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12029 return None;
12030 }
12031 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12032 return None;
12033 }
12034 let title = queue::title_from(raw, PR_TITLE_MAX);
12035 let lower = title.to_ascii_lowercase();
12036 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12037 return None;
12038 }
12039 Some(title)
12040}
12041
12042#[derive(Debug, Clone, PartialEq, Eq)]
12045struct BranchFacts {
12046 commits: Vec<(String, String)>,
12048 stat: String,
12050}
12051
12052const STAT_FILE_LINES: usize = 25;
12054const COMMITS_MAX_CHARS: usize = 20_000;
12056
12057async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12060 let commits = git::commit_log(repo, base, branch).await.ok()?;
12061 if commits.is_empty() {
12062 return None;
12063 }
12064 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12065 let lines: Vec<&str> = stat.lines().collect();
12066 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12067 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12068 let more = format!(" ... {omitted} more file(s)");
12069 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12070 kept.push(&more);
12071 kept.push(lines[lines.len() - 1]);
12072 kept.join("\n")
12073 } else {
12074 lines.join("\n")
12075 };
12076 Some(BranchFacts { commits, stat })
12077}
12078
12079fn markdown_safe(text: &str) -> String {
12082 text.replace("</details>", "</details>")
12083 .replace("\x60\x60\x60", "~~~")
12084}
12085
12086fn neutral_title(state: &RunState, winner: char) -> String {
12087 format!(
12088 "chore: land candidate {} of run {}",
12089 winner.to_ascii_uppercase(),
12090 state.id
12091 )
12092}
12093
12094pub(crate) async fn repair_landing_gate(
12097 state: &mut RunState,
12098 winner: &Candidate,
12099 outcomes: &[CommandOutcome],
12100) -> Result<GateFix> {
12101 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12102 return Ok(GateFix::Stop);
12103 }
12104 let mut runner = Runner {
12105 roles: state.config.resolve_roles()?,
12106 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12107 state: state.clone(),
12108 pause: Pause::new(),
12109 interrupt: Pause::new(),
12110 };
12111 let result = runner.gate_fix_round(winner, outcomes).await;
12112 *state = runner.state;
12113 result
12114}
12115
12116pub(crate) async fn verify_current_head(
12119 state: &mut RunState,
12120 winner: &Candidate,
12121 wait: &crate::repo_lock::WaitPolicy,
12122) -> Result<bool> {
12123 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12124 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12126 state
12127 .verification_history
12128 .push(crate::run::VerificationEvidence {
12129 superseded_by: Some(head.clone()),
12130 archived_at: Some(Timestamp::now()),
12131 fixes_spent: Some(state.gate_fixes.len()),
12132 gate_head: None,
12133 gate_ran: false,
12134 gate: Vec::new(),
12135 e2e: state.head_e2e.take(),
12136 });
12137 }
12138 if let Some(e) = &state.head_e2e
12139 && e.head == head
12140 {
12141 if e.passed {
12142 return Ok(true);
12143 }
12144 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12145 state.status = RunStatus::Blocked;
12146 state.save()?;
12147 return Ok(false);
12148 }
12149 } else if state.reviews.last().is_some_and(|r| {
12150 r.verified_head.as_deref() == Some(head.as_str())
12151 && (r.e2e_status() == E2eStatus::Passed
12152 || (r.e2e_status() == E2eStatus::NotConfigured
12153 && state.config.verify.e2e.is_empty()))
12154 }) || state
12155 .gate_repair_e2e
12156 .as_ref()
12157 .is_some_and(|e| e.head == head && e.passed)
12158 {
12159 return Ok(true);
12160 }
12161 if let Some(e) = &state.gate_repair_e2e
12162 && e.head == head
12163 && !e.passed
12164 && !e.outcomes.is_empty()
12165 && !verify_inconclusive(&e.outcomes)
12166 {
12167 state.status = RunStatus::Blocked;
12168 state.save()?;
12169 return Ok(false);
12170 }
12171 state.head_e2e = Some(crate::run::GateRepairE2e {
12172 head: head.clone(),
12173 passed: false,
12174 outcomes: Vec::new(),
12175 });
12176 state.save()?;
12177 let commands = state.config.verify.e2e.clone();
12178 let shell = state.config.shell();
12179 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12180 let cache_dir = state.config.cache_dir();
12181 let outcomes = if commands.is_empty() {
12182 Vec::new()
12183 } else {
12184 let (outcomes, _) = with_cache_lease(
12185 state,
12186 cache_dir.as_deref(),
12187 "e2e",
12188 "e2e",
12189 &winner.worktree,
12190 &head,
12191 timeout,
12192 "current head e2e",
12193 wait,
12194 |state, budget| async move {
12195 run_e2e_with_retry(
12196 state,
12197 &shell,
12198 &commands,
12199 &winner.worktree,
12200 budget,
12201 "current head e2e",
12202 )
12203 .await
12204 },
12205 )
12206 .await?;
12207 outcomes
12208 };
12209 let unresolved = verify_inconclusive(&outcomes);
12210 let passed = !unresolved
12211 && outcomes.iter().all(CommandOutcome::ok)
12212 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12213 state.head_e2e = Some(crate::run::GateRepairE2e {
12214 head: head.clone(),
12215 passed,
12216 outcomes,
12217 });
12218 if !passed && !unresolved {
12219 state.status = RunStatus::Blocked;
12220 state.event(
12221 "e2e",
12222 format!("verify.e2e failed on head {head}; nothing published"),
12223 );
12224 }
12225 state.save()?;
12226 Ok(passed)
12227}
12228
12229pub(crate) async fn check_gate_repair_e2e(
12232 state: &mut RunState,
12233 winner: &Candidate,
12234 wait: &crate::repo_lock::WaitPolicy,
12235) -> Result<GateFix> {
12236 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12237 return Ok(GateFix::Retry);
12238 }
12239 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12240 if state
12241 .gate_repair_e2e
12242 .as_ref()
12243 .is_some_and(|r| r.head == head && r.passed)
12244 {
12245 return Ok(GateFix::Retry);
12246 }
12247 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12248 head: head.clone(),
12249 passed: false,
12250 outcomes: Vec::new(),
12251 });
12252 state.save()?;
12253 let commands = state.config.verify.e2e.clone();
12254 let outcomes = if commands.is_empty() {
12255 Vec::new()
12256 } else {
12257 let shell = state.config.shell();
12258 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12259 let cache_dir = state.config.cache_dir();
12260 let (outcomes, _) = with_cache_lease(
12261 state,
12262 cache_dir.as_deref(),
12263 "e2e",
12264 "e2e",
12265 &winner.worktree,
12266 &head,
12267 timeout,
12268 "gate repair e2e",
12269 wait,
12270 |state, budget| async move {
12271 run_e2e_with_retry(
12272 state,
12273 &shell,
12274 &commands,
12275 &winner.worktree,
12276 budget,
12277 "gate repair e2e",
12278 )
12279 .await
12280 },
12281 )
12282 .await?;
12283 outcomes
12284 };
12285 let deferred = verify_inconclusive(&outcomes);
12286 let passed = outcomes.iter().all(CommandOutcome::ok)
12287 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12288 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12289 head,
12290 passed,
12291 outcomes,
12292 });
12293 state.save()?;
12294 if deferred {
12295 Ok(GateFix::Defer)
12296 } else if !passed {
12297 state.event(
12298 "gate",
12299 "verify.e2e failed after gate repair; nothing published",
12300 );
12301 Ok(GateFix::Stop)
12302 } else {
12303 Ok(GateFix::Retry)
12304 }
12305}
12306
12307pub(crate) async fn run_landing_gate(
12309 state: &mut RunState,
12310 winner: &Candidate,
12311 wait: &crate::repo_lock::WaitPolicy,
12312) -> Result<Vec<CommandOutcome>> {
12313 let shell = state.config.shell();
12314 let gate_commands = state.config.verify.gate.clone();
12315 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12316 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12317 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12318 Vec::new()
12319 } else {
12320 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12321 let cache_dir = state.config.cache_dir();
12322 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12323 let (outcomes, _) = with_cache_lease(
12324 state,
12325 cache_dir.as_deref(),
12326 "gate",
12327 "gate",
12328 &winner.worktree,
12329 &head,
12330 timeout,
12331 "final gate",
12332 wait,
12333 |state, budget| {
12334 let shell = shell.clone();
12335 let gate_commands = gate_commands.clone();
12336 let winner = winner.clone();
12337 async move {
12338 Runner::run_pre_gate(state, &winner).await;
12339 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12341 let (outcomes, timed_out_pids) = run_commands(
12342 state,
12343 "gate",
12344 "gate",
12345 0,
12346 &shell,
12347 &gate_commands,
12348 &winner.worktree,
12349 budget,
12350 )
12351 .await;
12352 (outcomes, false, timed_out_pids)
12353 }
12354 },
12355 )
12356 .await?;
12357 outcomes
12358 };
12359 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12360 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12361 let mut outcomes = outcomes;
12362 outcomes.push(CommandOutcome {
12363 command: "gate head stability".into(), code: None,
12364 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12365 duration_ms: 0, resource_blocked: false,
12366 });
12367 return Ok(outcomes);
12368 }
12369 if outcomes.is_empty() {
12370 state.event("gate", "no gate commands configured; not tested");
12375 }
12376 for o in &outcomes {
12377 state.event(
12378 "gate",
12379 format!(
12380 "`{}` -> {}",
12381 o.command,
12382 if o.ok() {
12383 "pass".to_owned()
12384 } else {
12385 format!(
12386 "FAIL ({:?})\n{}",
12387 o.code,
12388 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12389 )
12390 }
12391 ),
12392 );
12393 }
12394 Ok(outcomes)
12395}
12396
12397pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12403 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12404 ""
12405 } else {
12406 pr_title
12407 }
12408}
12409
12410pub fn landing_subject_source(state: &RunState) -> String {
12414 if is_review_run(state) {
12415 let winner = state.candidates.first().map_or('A', |c| c.label);
12416 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12417 }
12418 state.instruction.clone()
12419}
12420
12421fn summary_without_title(summary: &str) -> String {
12424 let mut lines = summary.trim().lines().peekable();
12425 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12426 lines.next();
12427 }
12428 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12429}
12430
12431#[cfg(test)]
12443fn pr_message(state: &RunState, winner: char) -> PrMessage {
12444 pr_message_with(state, winner, None)
12445}
12446
12447#[cfg(test)]
12450fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12451 let raw = pr_message_raw(
12452 state,
12453 winner,
12454 facts,
12455 crate::pr_outcome::FinalHead::Unreported,
12456 ChangeNote::NotAsked,
12457 );
12458 let id = crate::scrub::Identity::current();
12459 PrMessage {
12460 title: crate::scrub::scrub(&raw.title, &id),
12461 body: crate::scrub::scrub(&raw.body, &id),
12462 }
12463}
12464
12465#[derive(Debug, Clone, Copy)]
12467enum ChangeNote<'a> {
12468 NotAsked,
12470 Summary(&'a crate::run::ChangeSummary),
12472 Unavailable(&'a str),
12474}
12475
12476fn pr_message_raw(
12477 state: &RunState,
12478 winner: char,
12479 facts: Option<&BranchFacts>,
12480 head: crate::pr_outcome::FinalHead<'_>,
12481 change: ChangeNote<'_>,
12482) -> PrMessage {
12483 if !matches!(change, ChangeNote::NotAsked) {
12484 let (title, mut body) = match change {
12485 ChangeNote::Summary(c) => (
12486 c.title.clone(),
12487 format!("## Summary\n\n{}\n\n", c.body.trim()),
12488 ),
12489 ChangeNote::Unavailable(why) => (
12490 neutral_title(state, winner),
12491 format!(
12492 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12493 ),
12494 ),
12495 ChangeNote::NotAsked => unreachable!(),
12496 };
12497 body.push_str(&public_pr_evidence(state, head));
12498 body.push_str(&format!(
12499 "\n---\nmagi:run/{} magi:candidate-{}\n",
12500 state.id,
12501 winner.to_ascii_lowercase()
12502 ));
12503 return PrMessage { title, body };
12504 }
12505 pr_preview(state, winner, facts, head)
12506}
12507
12508fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12511 let mut body = crate::pr_outcome::review_rounds(state);
12512 body.push_str(&crate::pr_outcome::final_verification(state, head));
12513 let open = state.last_round_findings();
12514 if !open.is_empty() {
12515 body.push_str("## Open review findings\n\n");
12516 for f in open {
12517 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12518 }
12519 body.push('\n');
12520 }
12521 if let Some(c) = state.change_summary.as_ref()
12522 && head == crate::pr_outcome::FinalHead::At(&c.head)
12523 {
12524 body.push_str(&format!(
12525 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12526 c.base, c.head
12527 ));
12528 }
12529 body
12530}
12531
12532fn pr_preview(
12534 state: &RunState,
12535 winner: char,
12536 facts: Option<&BranchFacts>,
12537 head: crate::pr_outcome::FinalHead<'_>,
12538) -> PrMessage {
12539 let summary = state
12540 .candidates
12541 .iter()
12542 .find(|c| c.label == winner)
12543 .map(|c| c.summary.as_str())
12544 .unwrap_or_default();
12545 let review = is_review_run(state);
12548 let title = if review {
12549 facts
12550 .and_then(|f| {
12551 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12552 })
12553 .or_else(|| review_title(state))
12554 .or_else(|| {
12555 state
12556 .candidates
12557 .iter()
12558 .find(|c| c.label == winner)
12559 .filter(|c| !c.branch.starts_with("magi/"))
12560 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12561 })
12562 .unwrap_or_else(|| neutral_title(state, winner))
12563 } else {
12564 summary_title(summary).unwrap_or_else(|| {
12565 english_title(
12566 &state.instruction,
12567 PR_TITLE_MAX,
12568 state.config.graph.github_english(),
12569 )
12570 .unwrap_or_else(|| neutral_title(state, winner))
12571 })
12572 };
12573
12574 let mut body = String::new();
12575 let what = summary_without_title(summary);
12576 if !what.is_empty() {
12577 body.push_str("## Summary\n\n");
12578 body.push_str(&what);
12579 body.push_str("\n\n");
12580 }
12581
12582 body.push_str(&crate::pr_outcome::review_rounds(state));
12584 body.push_str(&crate::pr_outcome::final_verification(state, head));
12585 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12586
12587 let open = state.open_findings();
12588 if !open.is_empty() {
12589 body.push_str("## Open review findings\n\n");
12590 for f in &open {
12591 body.push_str(&format!(
12592 "- `{}` [{:?}] {}\n",
12593 f.id, f.severity, "Context retained in local review records."
12594 ));
12595 }
12596 body.push('\n');
12597 }
12598
12599 if let Some(fix) = fix
12600 && !fix.rejected.is_empty()
12601 {
12602 body.push_str("## Declined by the fixer\n\n");
12603 for r in &fix.rejected {
12604 body.push_str(&format!(
12605 "- `{}`: {}\n",
12606 r.id, "Reason retained in local review records."
12607 ));
12608 }
12609 body.push('\n');
12610 }
12611
12612 if review {
12613 body.push_str("## Commits under review\n\n");
12615 if let Some(facts) = facts {
12616 let mut left = COMMITS_MAX_CHARS;
12617 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12618 let mut entry = format!("- {}\n", markdown_safe(subject));
12619 for l in markdown_safe(text).lines() {
12620 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12621 }
12622 if left == 0 {
12623 body.push_str(&format!(
12624 "- ... {} more commit(s)\n",
12625 facts.commits.len() - i
12626 ));
12627 break;
12628 }
12629 if entry.len() > left {
12630 let mut end = left;
12633 while !entry.is_char_boundary(end) {
12634 end -= 1;
12635 }
12636 entry.truncate(end);
12637 entry.push_str("\n ... (truncated)\n");
12638 left = 0;
12639 } else {
12640 left -= entry.len();
12641 }
12642 body.push_str(&entry);
12643 }
12644 if !facts.stat.trim().is_empty() {
12645 body.push_str(&format!(
12646 "\n## Diff stat\n\n```\n{}\n```\n",
12647 markdown_safe(facts.stat.trim())
12648 ));
12649 }
12650 } else {
12651 match &state.reviewed_commits {
12652 Some(subjects) => {
12653 for s in subjects {
12654 body.push_str(&format!("- {}\n", s.trim()));
12655 }
12656 }
12657 None => {
12658 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12661 body.push_str(rest.trim());
12662 body.push('\n');
12663 }
12664 }
12665 }
12666 } else {
12667 let task = state.instruction.trim();
12668 let task = if task.is_empty() {
12669 "(empty task)"
12670 } else {
12671 task
12672 };
12673 body.push_str(&format!(
12674 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12675 task.replace("</details>", "</details>")
12676 ));
12677 }
12678
12679 body.push_str(&format!(
12680 "\n---\nmagi:run/{} magi:candidate-{}\n",
12681 state.id,
12682 winner.to_ascii_lowercase()
12683 ));
12684
12685 PrMessage { title, body }
12686}
12687
12688fn seeded_instruction(state: &RunState) -> String {
12691 let mut inputs = Vec::new();
12692 if let Some(start) = &state.sources.start_from {
12693 inputs.push(format!(
12694 "Start from `{}` at `{}` (including ancestry).",
12695 start.token, start.sha
12696 ));
12697 }
12698 for pick in &state.sources.pick_commits {
12699 inputs.push(format!(
12700 "Pick `{}` at `{}` (single commit).",
12701 pick.token, pick.sha
12702 ));
12703 }
12704 if inputs.is_empty() {
12705 state.instruction.clone()
12706 } else {
12707 format!(
12708 "{}\n\n# Explicit source inputs already prepared\n\n{}",
12709 state.instruction,
12710 inputs.join("\n")
12711 )
12712 }
12713}
12714
12715async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12719 let base = &state.base_branch;
12720 let mut against = base.clone();
12721 if mode == MergeMode::Pr {
12722 let remote = &state.config.merge.remote;
12726 let tracking = format!("{remote}/{base}");
12727 let fetched = git::fetch(repo, remote, base).await;
12728 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12729 against = tracking;
12730 } else {
12731 return false;
12732 }
12733 }
12734 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12735}
12736
12737fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12740 let mut detail = format!(
12741 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12742 nothing to open a pull request for"
12743 );
12744 detail.push_str(
12745 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12746 );
12747 detail
12748}
12749
12750#[derive(Debug, PartialEq, Eq)]
12753enum PrPlan {
12754 Create,
12755 Adopt { url: String, title: String },
12756 Stop(String),
12757}
12758
12759fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12762 match found {
12763 Ok(land::OpenPr::None) => PrPlan::Create,
12764 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12765 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12766 "several open pull requests exist for this branch, not picking one: {}",
12767 urls.join(" ")
12768 )),
12769 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12770 }
12771}
12772
12773pub(crate) async fn gh_pr_create(
12775 cwd: &Path,
12776 base: &str,
12777 head: &str,
12778 title: &str,
12779 body: &str,
12780) -> Result<String> {
12781 let out = tokio::process::Command::new("gh")
12782 .args([
12783 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12784 ])
12785 .current_dir(cwd)
12786 .quiet()
12787 .stdin(std::process::Stdio::null())
12788 .output()
12789 .await
12790 .context("spawn gh")?;
12791 if out.status.success() {
12792 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12793 } else {
12794 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12795 }
12796}
12797
12798async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12800 match std::fs::symlink_metadata(path) {
12801 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12802 git::worktree_remove_clean(repo, path).await?;
12805 return Ok(());
12806 }
12807 Err(e) => return Err(e.into()),
12808 Ok(_) => {}
12809 }
12810 if !git::is_clean(path).await? {
12811 bail!(
12812 "worktree has uncommitted work; refusing removal: {}",
12813 path.display()
12814 );
12815 }
12816 if !git::worktree_remove_clean(repo, path).await? {
12817 bail!("worktree removal refused: {}", path.display());
12818 }
12819 Ok(())
12820}
12821
12822pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12831 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12832 bail!("refusing fold: {reason}");
12833 }
12834 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12835 bail!("candidate preservation incomplete; refusing cleanup");
12836 }
12837 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12841 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12842 let _driver = crate::driver::maintenance(home, &state.id)?;
12843 let mut output_guards = Vec::new();
12844 for entry in std::fs::read_dir(state.worktree_root())
12845 .into_iter()
12846 .flatten()
12847 .flatten()
12848 {
12849 if entry.path().join(".git").is_file() {
12850 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12851 }
12852 }
12853 let repo = state.repo.clone();
12854 let wait = repository_wait(state, None);
12857 let _coordination = match coordinate_operation(state, &repo, &wait).await {
12858 Ok(lease) => lease,
12859 Err(e) => {
12860 state.save_under(home)?;
12861 return Err(e);
12862 }
12863 };
12864 let _source_outputs = if state.status == RunStatus::Merged
12867 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12868 {
12869 verify_merged_fold_sources(state, drop_winner, home).await?
12870 } else {
12871 Vec::new()
12872 };
12873 let root = state.worktree_root();
12874 let winner = state.tally.as_ref().map(|t| t.winner);
12875 let mut removed = Vec::new();
12876
12877 for i in 0..state.candidates.len() {
12878 let c = state.candidates[i].clone();
12879 let is_winner = Some(c.label) == winner;
12880 if is_winner && !drop_winner {
12881 continue;
12882 }
12883 let existed = c.worktree.exists();
12884 remove_preserved_worktree(&repo, &c.worktree).await?;
12887 if existed {
12888 removed.push(c.worktree.to_string_lossy().into_owned());
12889 }
12890 let handed_over = state.released_branches.contains(&c.branch);
12893 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12894 if !git::branch_delete(&repo, &c.branch).await? {
12895 bail!("candidate branch removal refused");
12896 }
12897 removed.push(c.branch.clone());
12898 }
12899 state.candidates[i].folded = true;
12900 }
12901
12902 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12903 let path = name.path();
12904 let keep = !drop_winner
12905 && winner.is_some_and(|w| {
12906 path.file_name()
12907 .is_some_and(|n| n == format!("cand-{w}").as_str())
12908 });
12909 if keep {
12910 continue;
12911 }
12912 remove_preserved_worktree(&repo, &path).await?;
12913 removed.push(path.to_string_lossy().into_owned());
12914 }
12915
12916 remove_if_empty(&root);
12925
12926 if state.enabled_worktree_config && drop_winner {
12927 git::release_worktree_config(&repo).await.ok();
12931 state.enabled_worktree_config = false;
12932 }
12933 state.save_under(home)?;
12934 Ok(removed)
12935}
12936
12937async fn verify_merged_fold_sources(
12938 state: &RunState,
12939 drop_winner: bool,
12940 home: &Path,
12941) -> Result<Vec<std::fs::File>> {
12942 let winner = state.tally.as_ref().map(|t| t.winner);
12943 let mut heads = BTreeSet::new();
12944 for candidate in &state.candidates {
12945 if Some(candidate.label) == winner && !drop_winner {
12946 continue;
12947 }
12948 anyhow::ensure!(
12949 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12950 "candidate recovery source is outside the inventoried run root: {}",
12951 candidate.worktree.display()
12952 );
12953 match std::fs::symlink_metadata(&candidate.worktree) {
12954 Ok(_) => {}
12955 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12956 Err(error) => return Err(error.into()),
12957 }
12958 if !state.released_branches.contains(&candidate.branch)
12959 && git::branch_exists(&state.repo, &candidate.branch).await?
12960 {
12961 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12962 }
12963 }
12964 let mut cargo_guards = Vec::new();
12965 let entries = match std::fs::read_dir(state.worktree_root()) {
12966 Ok(entries) => Some(entries),
12967 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12968 Err(error) => return Err(error.into()),
12969 };
12970 for entry in entries.into_iter().flatten() {
12971 let path = entry?.path();
12972 let metadata = std::fs::symlink_metadata(&path)?;
12973 anyhow::ensure!(
12974 metadata.is_dir() && !metadata.file_type().is_symlink(),
12975 "non-directory or symlink recovery source retained: {}",
12976 path.display()
12977 );
12978 if !drop_winner
12979 && winner.is_some_and(|w| {
12980 path.file_name()
12981 .is_some_and(|name| name == format!("cand-{w}").as_str())
12982 })
12983 {
12984 continue;
12985 }
12986 anyhow::ensure!(
12987 path.join(".git").is_file(),
12988 "unregistered recovery source retained: {}",
12989 path.display()
12990 );
12991 anyhow::ensure!(
12992 git::git(
12993 &path,
12994 &[
12995 "status",
12996 "--porcelain",
12997 "--untracked-files=all",
12998 "--ignored",
12999 "--",
13000 ".",
13001 ":(exclude)target"
13002 ]
13003 )
13004 .await?
13005 .is_empty(),
13006 "source changes or ignored recovery files retained: {}",
13007 path.display()
13008 );
13009 match std::fs::symlink_metadata(path.join("target")) {
13010 Ok(_) => {
13011 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13012 }
13013 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13014 Err(error) => return Err(error.into()),
13015 }
13016 for checkpoint in [
13017 "rebase-merge",
13018 "rebase-apply",
13019 "sequencer",
13020 "CHERRY_PICK_HEAD",
13021 "MERGE_HEAD",
13022 "REBASE_HEAD",
13023 "REVERT_HEAD",
13024 "BISECT_LOG",
13025 "AUTO_MERGE",
13026 "MERGE_MSG",
13027 ] {
13028 let location = git::git(
13029 &path,
13030 &[
13031 "rev-parse",
13032 "--path-format=absolute",
13033 "--git-path",
13034 checkpoint,
13035 ],
13036 )
13037 .await?;
13038 anyhow::ensure!(
13039 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13040 if e.kind() == std::io::ErrorKind::NotFound),
13041 "Git recovery checkpoint retained: {checkpoint}"
13042 );
13043 }
13044 heads.insert(git::rev_parse(&path, "HEAD").await?);
13045 }
13046 crate::fold_evidence::preserve(state, home, &heads).await?;
13047
13048 Ok(cargo_guards)
13049}
13050
13051fn remove_if_empty(dir: &Path) {
13062 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13063 std::fs::remove_dir(dir).ok();
13064 }
13065}
13066
13067pub fn worst_open(state: &RunState) -> Option<Severity> {
13069 state
13070 .reviews
13071 .last()?
13072 .reviews
13073 .iter()
13074 .flat_map(|r| r.findings.iter())
13075 .map(|f| f.severity)
13076 .max()
13077}
13078
13079async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13083 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13084 git::acquire_worktree_config(repo).await?;
13085 state.enabled_worktree_config = true;
13086 }
13087 Ok(())
13088}
13089
13090#[cfg(test)]
13091mod tests {
13092 #[tokio::test]
13093 async fn launcher_base_fetch_requires_remote_success() {
13094 let tmp = tempfile::tempdir().unwrap();
13095 let repo = tmp.path().join("repo");
13096 let remote = tmp.path().join("remote");
13097 std::fs::create_dir(&repo).unwrap();
13098 std::fs::create_dir(&remote).unwrap();
13099 git::git(&remote, &["init", "--bare", "-b", "main"])
13100 .await
13101 .unwrap();
13102 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13103 git::git(
13104 &repo,
13105 &[
13106 "-c",
13107 "user.name=Fixture",
13108 "-c",
13109 "user.email=fixture@example.invalid",
13110 "commit",
13111 "--allow-empty",
13112 "-m",
13113 "fixture",
13114 ],
13115 )
13116 .await
13117 .unwrap();
13118 git::git(
13119 &repo,
13120 &["remote", "add", "origin", remote.to_str().unwrap()],
13121 )
13122 .await
13123 .unwrap();
13124 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13125 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13126 assert_eq!(
13127 resolve_base(&repo, "main", "origin", None)
13128 .await
13129 .unwrap()
13130 .commit,
13131 head
13132 );
13133 #[cfg(unix)]
13134 {
13135 let policy: crate::config::Coordination = toml::from_str(
13136 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13137 ).unwrap();
13138 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13139 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13140 let held = crate::repo_lock::SeatLease::operation_with(
13141 &repo,
13142 Some(&policy),
13143 &crate::repo_lock::WaitPolicy::default(),
13144 )
13145 .await
13146 .unwrap();
13147 assert!(held.is_some());
13148 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13149 tokio::pin!(fetch);
13150 assert!(
13151 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13152 .await
13153 .is_err()
13154 );
13155 assert!(
13156 !repo.join(".git/FETCH_HEAD").exists(),
13157 "no fetch before admission"
13158 );
13159 drop(held);
13160 let fetched = fetch.await.unwrap();
13161 assert_eq!(fetched.commit, head);
13162 assert_eq!(fetched.remote, "origin");
13163 assert_eq!(fetched.branch, "main");
13164 }
13165 git::git(
13167 &repo,
13168 &[
13169 "remote",
13170 "set-url",
13171 "origin",
13172 tmp.path().join("missing").to_str().unwrap(),
13173 ],
13174 )
13175 .await
13176 .unwrap();
13177 let error = resolve_base(&repo, "main", "origin", None)
13178 .await
13179 .unwrap_err()
13180 .to_string();
13181 assert!(error.contains("cannot read origin/main"));
13182 assert!(error.contains("refusing to branch off"));
13183 }
13184
13185 #[test]
13186 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13187 let leaked = vec!["chore(deps): update a crate".to_owned()];
13188 let own = "fix(daemon): apply a chosen action";
13189 assert!(should_retitle("", own, &leaked));
13190 assert!(should_retitle(
13191 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13192 own,
13193 &leaked
13194 ));
13195 assert!(should_retitle(
13196 "chore: land candidate A of run 1",
13197 own,
13198 &leaked
13199 ));
13200 assert!(should_retitle(
13201 "magi: candidate A (uncommitted work)",
13202 own,
13203 &leaked
13204 ));
13205 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13206 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13207 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13208 }
13209
13210 #[test]
13211 fn pr_merge_plan_creates_adopts_or_stops() {
13212 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13213 assert_eq!(
13214 pr_merge_plan(Ok(land::OpenPr::One {
13215 url: "u".into(),
13216 title: "t".into()
13217 })),
13218 PrPlan::Adopt {
13219 url: "u".into(),
13220 title: "t".into()
13221 }
13222 );
13223 let PrPlan::Stop(many) =
13224 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13225 else {
13226 panic!("many must stop");
13227 };
13228 assert!(many.contains('a') && many.contains('b'));
13229 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13230 panic!("a failed lookup must stop");
13231 };
13232 assert!(err.contains("bad token"));
13233 }
13234
13235 use super::*;
13236 use crate::run::GateStatus;
13237 use std::collections::BTreeMap;
13238 use std::time::Duration;
13239
13240 fn conductor() -> AgentSpec {
13241 AgentSpec {
13242 id: "conductor".to_owned(),
13243 kind: crate::config::AgentKind::Command,
13244 model: None,
13245 command: vec!["true".to_owned()],
13246 extra_args: Vec::new(),
13247 env: BTreeMap::new(),
13248 prompt_delivery: None,
13249 max_turn_secs: None,
13250 }
13251 }
13252
13253 fn spec(id: &str) -> AgentSpec {
13254 AgentSpec {
13255 id: id.to_owned(),
13256 kind: crate::config::AgentKind::Command,
13257 model: None,
13258 command: vec!["true".to_owned()],
13259 extra_args: Vec::new(),
13260 env: BTreeMap::new(),
13261 prompt_delivery: None,
13262 max_turn_secs: None,
13263 }
13264 }
13265
13266 fn ids(xs: &[&str]) -> BTreeSet<String> {
13267 xs.iter().map(|s| (*s).to_owned()).collect()
13268 }
13269
13270 #[test]
13271 fn pick_successor_skips_an_agent_another_seat_holds() {
13272 let roster = [spec("a"), spec("b"), spec("c")];
13274 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13275 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13276 }
13277
13278 #[test]
13279 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13280 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13282 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13283 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13284 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13285 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13286 }
13287
13288 #[test]
13289 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13290 let roster = [spec("a"), spec("b")];
13291 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13292 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13293 let carried = ids(&["b"]);
13294 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13295 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13296 }
13297
13298 #[test]
13299 fn pick_successor_returns_none_without_any_untried_successor() {
13300 let roster = [spec("a"), spec("b")];
13301 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13302 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13303 let carried = ids(&["a"]);
13304 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13305 }
13306
13307 #[test]
13308 fn pick_successor_rescue_avoids_another_seats_occupant() {
13309 let roster = [spec("a"), spec("b"), spec("c")];
13311 let carried = ids(&["b", "c"]);
13312 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13313 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13314 }
13315
13316 #[test]
13317 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13318 let roster = [spec("a"), spec("b"), spec("c")];
13319 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13320 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13321 }
13322
13323 #[test]
13324 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13325 let roster = [spec("a"), spec("b"), spec("c")];
13326 let failed = ids(&["a", "b", "c"]);
13327 let mut tried = ids(&["b"]);
13329 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13330 assert_eq!(first.id, "a");
13331 tried.insert(first.id.clone());
13332 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13333 assert_eq!(second.id, "c");
13334 tried.insert(second.id.clone());
13335 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13336 }
13337
13338 #[test]
13339 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13340 let roster = [spec("a"), spec("b")];
13341 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13342 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13343 }
13344
13345 #[test]
13346 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13347 let roster = [spec("a"), spec("b"), spec("c")];
13348 let h = SeatHistory {
13349 failed: ids(&["a"]),
13350 last_ok: Some("b".to_owned()),
13351 last_fail: None,
13352 };
13353 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13354 let gone = SeatHistory {
13356 last_ok: Some("zzz".to_owned()),
13357 ..h.clone()
13358 };
13359 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13360 let failed = SeatHistory {
13361 failed: ids(&["a", "b"]),
13362 last_ok: Some("b".to_owned()),
13363 last_fail: None,
13364 };
13365 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13366 }
13367
13368 #[test]
13369 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13370 let first = SeatState::new("review-1", "alpha", 7);
13371 let next = handover_seat("review-1", "gamma", 7);
13372 assert_ne!(first.claude_session, next.claude_session);
13373 }
13374
13375 #[test]
13376 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13377 let mut state = state_with_summary("x", "y");
13378 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13379 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13380 assert_ne!(a.claude_session, b.claude_session);
13381 }
13382
13383 #[test]
13384 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13385 let roster = [spec("a"), spec("b")];
13386 let h = SeatHistory {
13387 failed: ids(&["a", "b"]),
13388 ..SeatHistory::default()
13389 };
13390 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13391 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13392 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13393 }
13394
13395 #[test]
13402 fn failure_signature_ignores_numbers_and_paths() {
13403 assert_eq!(
13404 failure_signature("exited with Some(2) and no usable output"),
13405 failure_signature("exited with Some(137) and no usable output")
13406 );
13407 assert_eq!(
13408 failure_signature("process exited with no usable output (exit code 2)"),
13409 failure_signature("process exited with no usable output (exit code 137)")
13410 );
13411 for (old, new) in [
13412 (
13413 "exited with Some(2) and no usable output",
13414 "process exited with no usable output (exit code 2)",
13415 ),
13416 (
13417 "exited with None and no usable output",
13418 "process exited with no usable output (exit code unavailable)",
13419 ),
13420 ] {
13421 let previous = FailClass::Other(failure_signature(old));
13422 let current = FailClass::Other(failure_signature(new));
13423 assert_eq!(previous, current, "resumption retains the failure class");
13424 assert!(!should_hand_over(Some(&previous), ¤t));
13425 }
13426 assert_eq!(
13427 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13428 failure_signature("cannot open /var/x.txt: denied")
13429 );
13430 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13431 }
13432
13433 #[test]
13434 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13435 let other = FailClass::Other("x".into());
13436 assert!(should_hand_over(None, &FailClass::Quota));
13437 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13438 assert!(should_hand_over(
13439 Some(&FailClass::Timeout),
13440 &FailClass::Timeout
13441 ));
13442 assert!(should_hand_over(None, &other));
13443 assert!(!should_hand_over(Some(&other), &other));
13444 assert!(should_hand_over(
13445 Some(&other),
13446 &FailClass::Other("y".into())
13447 ));
13448 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13450 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13451 }
13452
13453 #[test]
13454 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13455 let a = SeatState::new("judge-1", "alpha", 7);
13456 let b = handover_seat("judge-1", "beta", 7);
13457 assert_ne!(a.claude_session, b.claude_session);
13458 assert_eq!(b.turns, 0);
13459 }
13460
13461 #[test]
13462 fn a_timeout_is_classified_apart_from_other_failures() {
13463 assert_eq!(
13464 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13465 Some(FailClass::Timeout)
13466 );
13467 assert!(matches!(
13468 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13469 Some(FailClass::Other(_))
13470 ));
13471 }
13472
13473 #[test]
13474 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13475 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13476 let tried = BTreeSet::from(["beta".to_owned()]);
13477 let next = next_untried_in_roster(&roster, 1, &tried);
13480 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13481 }
13482
13483 #[test]
13484 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13485 let roster = vec![spec("alpha"), spec("beta")];
13486 let tried = BTreeSet::from(["beta".to_owned()]);
13487 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13491 }
13492
13493 #[test]
13494 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13495 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13496 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13497 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13501 }
13502
13503 #[test]
13504 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13505 let roster = vec![spec("a"), spec("a"), spec("b")];
13506 let tried = BTreeSet::from(["a".to_owned()]);
13507 let next = next_untried_in_roster(&roster, 0, &tried);
13508 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13509 }
13510
13511 #[test]
13512 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13513 let roster = vec![spec("a"), spec("b")];
13514 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13515 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13516 }
13517
13518 #[test]
13519 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13520 let dir = tempfile::tempdir().unwrap();
13521 let bay = dir.path().join("ffff");
13522
13523 remove_if_empty(&bay);
13525 assert!(!bay.exists());
13526
13527 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13530 remove_if_empty(&bay);
13531 assert!(bay.exists(), "non-empty directory must survive");
13532
13533 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13535 remove_if_empty(&bay);
13536 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13537 }
13538
13539 #[test]
13548 fn a_full_panel_that_found_nothing_is_clean() {
13549 assert!(round_is_clean(
13550 0,
13551 true,
13552 2,
13553 2,
13554 0,
13555 IncompleteReviewPolicy::Block
13556 ));
13557 }
13558
13559 #[test]
13560 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13561 assert!(!round_is_clean(
13562 0,
13563 true,
13564 1,
13565 2,
13566 0,
13567 IncompleteReviewPolicy::Block
13568 ));
13569 }
13570
13571 #[test]
13572 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13573 assert!(!round_is_clean(
13574 1,
13575 true,
13576 1,
13577 2,
13578 0,
13579 IncompleteReviewPolicy::Warn
13580 ));
13581 }
13582
13583 #[test]
13584 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13585 assert!(round_is_clean(
13586 0,
13587 true,
13588 1,
13589 2,
13590 0,
13591 IncompleteReviewPolicy::Warn
13592 ));
13593 }
13594
13595 #[test]
13596 fn a_full_panel_with_an_open_finding_is_not_clean() {
13597 assert!(!round_is_clean(
13598 1,
13599 true,
13600 2,
13601 2,
13602 0,
13603 IncompleteReviewPolicy::Block
13604 ));
13605 }
13606
13607 #[test]
13608 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13609 assert!(!round_is_clean(
13610 0,
13611 false,
13612 2,
13613 2,
13614 0,
13615 IncompleteReviewPolicy::Block
13616 ));
13617 }
13618
13619 #[test]
13626 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13627 assert!(round_is_clean(
13630 0,
13631 true,
13632 1,
13633 2,
13634 1,
13635 IncompleteReviewPolicy::Block
13636 ));
13637 }
13638
13639 #[test]
13640 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13641 assert!(!round_is_clean(
13644 0,
13645 true,
13646 1,
13647 2,
13648 0,
13649 IncompleteReviewPolicy::Block
13650 ));
13651 }
13652
13653 #[test]
13654 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13655 assert!(!round_is_clean(
13656 1,
13657 true,
13658 1,
13659 2,
13660 1,
13661 IncompleteReviewPolicy::Block
13662 ));
13663 assert!(!round_is_clean(
13664 0,
13665 false,
13666 1,
13667 2,
13668 1,
13669 IncompleteReviewPolicy::Block
13670 ));
13671 }
13672
13673 #[test]
13674 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13675 assert!(!round_is_clean(
13679 0,
13680 true,
13681 0,
13682 2,
13683 2,
13684 IncompleteReviewPolicy::Block
13685 ));
13686 }
13687
13688 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13689 CommandOutcome {
13690 command: "test".to_owned(),
13691 code,
13692 output_tail: String::new(),
13693 duration_ms: 0,
13694 resource_blocked,
13695 }
13696 }
13697
13698 #[test]
13699 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13700 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13701 assert!(
13702 !verify_inconclusive(&[outcome(Some(1), false)]),
13703 "an ordinary failure is still evidence about the patch"
13704 );
13705 assert!(verify_inconclusive(&[outcome(None, true)]));
13706 assert!(
13707 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13708 "one inconclusive outcome taints the whole batch"
13709 );
13710 assert!(!verify_inconclusive(&[]));
13711 }
13712
13713 #[tokio::test]
13714 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13715 let calls = std::sync::atomic::AtomicUsize::new(0);
13719 let started = Instant::now();
13720 wait_for_pids_with(
13721 &[123],
13722 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13723 Duration::from_millis(5),
13724 Duration::from_secs(5),
13725 )
13726 .await;
13727 assert!(
13728 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13729 "must keep checking rather than deciding on the first answer"
13730 );
13731 assert!(
13732 started.elapsed() < Duration::from_secs(1),
13733 "must return the moment it is confirmed dead, not wait out the ceiling"
13734 );
13735 }
13736
13737 #[tokio::test]
13738 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13739 let started = Instant::now();
13740 wait_for_pids_with(
13741 &[123],
13742 |_| true, Duration::from_millis(5),
13744 Duration::from_millis(30),
13745 )
13746 .await;
13747 let elapsed = started.elapsed();
13748 assert!(
13749 elapsed >= Duration::from_millis(30),
13750 "must not give up before its own ceiling: {elapsed:?}"
13751 );
13752 assert!(
13753 elapsed < Duration::from_secs(1),
13754 "must not wait past its own ceiling either: {elapsed:?}"
13755 );
13756 }
13757
13758 #[tokio::test]
13759 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13760 let started = Instant::now();
13761 wait_for_pids_with(
13762 &[],
13763 |_| true,
13764 Duration::from_secs(5),
13765 Duration::from_secs(5),
13766 )
13767 .await;
13768 assert!(
13769 started.elapsed() < Duration::from_millis(200),
13770 "an empty pid list has nothing to confirm"
13771 );
13772 }
13773
13774 fn review_round(
13780 clean: bool,
13781 blocking: usize,
13782 answered: usize,
13783 expected: usize,
13784 progressed: bool,
13785 e2e_ok: bool,
13786 ) -> ReviewRound {
13787 ReviewRound {
13788 round: 1,
13789 head: "h".to_owned(),
13790 verified_head: None,
13791 verified_at: None,
13792 reviews: Vec::new(),
13793 e2e: vec![CommandOutcome {
13794 command: "test".to_owned(),
13795 code: Some(if e2e_ok { 0 } else { 1 }),
13796 output_tail: String::new(),
13797 duration_ms: 0,
13798 resource_blocked: false,
13799 }],
13800 verify_retried: false,
13801 e2e_deferred: false,
13802 e2e_defer_reason: None,
13803 fix: None,
13804 blocking,
13805 answered,
13806 expected,
13807 clean,
13808 progressed,
13809 vote_split: false,
13810 reconsideration: Vec::new(),
13811 verdict: None,
13812 }
13813 }
13814
13815 #[test]
13816 fn review_conclusion_is_none_when_nothing_has_run() {
13817 assert_eq!(review_conclusion(&[], 3), None);
13818 }
13819
13820 #[test]
13821 fn review_conclusion_is_none_while_rounds_remain() {
13822 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13823 assert_eq!(review_conclusion(&rounds, 3), None);
13824 }
13825
13826 #[test]
13827 fn review_conclusion_is_gating_once_a_round_is_clean() {
13828 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13829 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13830 }
13831
13832 #[test]
13833 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13834 let rounds = vec![
13835 review_round(false, 1, 2, 2, true, true),
13836 review_round(false, 1, 2, 2, true, true),
13837 ];
13838 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13839 }
13840
13841 #[test]
13842 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13843 let rounds = vec![
13844 review_round(false, 1, 2, 2, true, true),
13845 review_round(false, 1, 2, 2, true, false),
13846 ];
13847 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13848 }
13849
13850 #[test]
13851 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13852 let mut blocked = review_round(false, 1, 2, 2, true, false);
13859 blocked.e2e[0].resource_blocked = true;
13860 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13861 assert_eq!(review_conclusion(&rounds, 2), None);
13862 }
13863
13864 #[test]
13865 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13866 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13868 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13869 }
13870
13871 #[test]
13872 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13873 let rounds = vec![
13874 review_round(false, 1, 2, 2, false, true),
13875 review_round(false, 1, 2, 2, false, true),
13876 ];
13877 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13878 }
13879
13880 fn secs(n: u64) -> Duration {
13881 Duration::from_secs(n)
13882 }
13883
13884 #[tokio::test]
13887 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13888 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13889 if std::env::var_os(CHILD).is_none() {
13890 let output = std::process::Command::new(std::env::current_exe().unwrap())
13894 .args([
13895 "--exact",
13896 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13897 "--nocapture",
13898 ])
13899 .env(CHILD, "1")
13900 .env_remove("MAGI_WORKER_HOME")
13901 .env_remove("MAGI_WORKER_ID")
13902 .output()
13903 .unwrap();
13904 assert!(
13905 output.status.success()
13906 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13907 "{}\n{}",
13908 String::from_utf8_lossy(&output.stdout),
13909 String::from_utf8_lossy(&output.stderr)
13910 );
13911 return;
13912 }
13913 ask_test_home();
13914 let tmp = tempfile::tempdir().unwrap();
13915 let repo = tmp.path().join("repo");
13916 std::fs::create_dir_all(&repo).unwrap();
13917 init_repo(&repo);
13918 let g = |args: &[&str]| {
13919 let out = std::process::Command::new("git")
13920 .args(args)
13921 .current_dir(&repo)
13922 .quiet()
13923 .output()
13924 .unwrap();
13925 assert!(
13926 out.status.success(),
13927 "{}",
13928 String::from_utf8_lossy(&out.stderr)
13929 );
13930 };
13931 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13932 g(&["checkout", "-b", "historical"]);
13933 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13934 g(&["add", "."]);
13935 g(&["commit", "-m", "old alternative"]);
13936 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13937 g(&["checkout", "-b", "magi/task-selected", &base]);
13938 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13939 g(&["add", "."]);
13940 g(&["commit", "-m", "measurement infrastructure"]);
13941 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13942 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13943 g(&["add", "."]);
13944 g(&["commit", "-m", "dependent implementation"]);
13945 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13946 g(&["checkout", "main"]);
13947 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13948 g(&["add", "."]);
13949 g(&["commit", "-m", "unrelated base change"]);
13950 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13951 let origin = tmp.path().join("origin.git");
13952 g(&[
13953 "clone",
13954 "--bare",
13955 repo.to_str().unwrap(),
13956 origin.to_str().unwrap(),
13957 ]);
13958 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13959 g(&["fetch", "origin"]);
13960 let selection = crate::pr_recovery::Selection {
13961 url: "https://example.test/project/pull/1".into(),
13962 branch: "magi/task-selected".into(),
13963 head: head.clone(),
13964 source_head: None,
13965 source_run: None,
13966 source_proof: None,
13967 disposition: Some(crate::pr_recovery::Disposition::Review),
13968 instruction: "preserve measured behavior".into(),
13969 approved_at: jiff::Timestamp::now(),
13970 };
13971 let mut template = runner_at(RunStatus::Reviewing);
13972 template.state.repo = repo.clone();
13973 template.state.base_commit = current.clone();
13974 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13975 template.state.origin = Some(crate::run::Origin::queue("selected"));
13976 template.state.pr_head = Some(selection.branch.clone());
13977 template.state.pr_recovery = Some(selection.clone());
13978 let mut runner =
13979 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13980 .await
13981 .unwrap();
13982 let wt = runner.state.winner().unwrap().worktree.clone();
13983 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13984 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13985 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13986 assert!(runner.state.seeds.is_empty());
13987 assert_eq!(runner.state.instruction, "preserve measured behavior");
13988 runner.sync_to_base().await.unwrap();
13989 assert_eq!(
13990 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13991 "prerequisite"
13992 );
13993 assert_eq!(
13994 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13995 "measured implementation"
13996 );
13997 assert_eq!(
13998 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13999 "current main change"
14000 );
14001 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14002 assert_eq!(
14003 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14004 head
14005 );
14006
14007 let mut partial = runner_at(RunStatus::Reviewing);
14009 partial.state.repo = repo.clone();
14010 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14011 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14012 run: Some("exit 1".into()),
14013 ..Default::default()
14014 }];
14015 let selection = runner.state.pr_recovery.clone().unwrap();
14016 partial.state.pr_recovery = Some(selection.clone());
14017 partial.state.pr_head = Some(selection.branch.clone());
14018 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14019 let id = partial.state.id.clone();
14020 let error = match Runner::open_selected(
14021 &repo,
14022 partial.state,
14023 partial.roles,
14024 1,
14025 runner.state.base_commit.clone(),
14026 selection,
14027 )
14028 .await
14029 {
14030 Ok(_) => panic!("setup must fail"),
14031 Err(e) => e,
14032 };
14033 assert_eq!(
14034 error
14035 .downcast_ref::<crate::run::StartupFailure>()
14036 .unwrap()
14037 .run,
14038 id
14039 );
14040 let failed = RunState::load(&id).unwrap();
14041 assert_eq!(failed.status, RunStatus::Failed);
14042 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14043 assert!(Runner::resume(&id).is_err());
14044 runner.state.status = RunStatus::Ready;
14045 runner.state.driver_exited = true;
14046 runner.state.save().unwrap();
14047 let mut retry = runner_at(RunStatus::Reviewing);
14048 retry.state.repo = repo.clone();
14049 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14050 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14051 reselected.approved_at = jiff::Timestamp::now();
14052 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14053 retry.state.pr_head = Some(reselected.branch.clone());
14054 retry.state.pr_recovery = Some(reselected.clone());
14055 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14056 .await
14057 .unwrap();
14058 let restarted = Runner::open_selected(
14059 &repo,
14060 retry.state,
14061 retry.roles,
14062 1,
14063 runner.state.base_commit.clone(),
14064 reselected,
14065 )
14066 .await
14067 .unwrap();
14068 assert_eq!(
14069 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14070 .await
14071 .unwrap(),
14072 head
14073 );
14074 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14075 assert_eq!(
14076 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14077 head
14078 );
14079 }
14080
14081 #[tokio::test]
14082 #[cfg(unix)]
14083 async fn fallback_and_interrupted_restoration_require_coordination() {
14084 use std::os::fd::AsRawFd as _;
14085 ask_test_home();
14086 let tmp = tempfile::tempdir().unwrap();
14087 let mut runner = registered_runner(tmp.path()).await;
14088 runner.state.config.graph.timeout_resource_wait = 1;
14089 let winner = runner.state.candidates[0].clone();
14090 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14091 let leftover = winner.worktree.join("leftover");
14092 std::fs::write(&leftover, "untrusted edits").unwrap();
14093 let next = tmp.path().join("next-fixer");
14094 let mut failed = spec("first");
14095 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14096 let mut successor = spec("second");
14097 successor.command = vec![
14098 "sh".into(),
14099 "-c".into(),
14100 format!("touch '{}'; echo done", next.display()),
14101 ];
14102 let attempts = vec![
14103 (failed.clone(), "first".into()),
14104 (successor.clone(), "second".into()),
14105 ];
14106 let lock_path = runner.state.repo.join(".git/updater.lock");
14107 let external = std::fs::File::create(&lock_path).unwrap();
14108 assert_eq!(
14109 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14110 0
14111 );
14112 let artifacts = tmp.path().join("artifacts");
14113 let build = |spec, seat| SeatJob {
14114 spec,
14115 seat,
14116 cwd: winner.worktree.clone(),
14117 prompt: "fixture".into(),
14118 timeout: Duration::from_secs(10),
14119 allow_write: true,
14120 sessions: false,
14121 artifacts: artifacts.clone(),
14122 stem: "fix".into(),
14123 handover: None,
14124 };
14125 let (_, _, out) = runner
14126 .ask_fixer_among(
14127 attempts.clone(),
14128 "graft",
14129 None,
14130 Some((&winner.worktree, &head)),
14131 build,
14132 )
14133 .await;
14134 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14135 assert!(leftover.exists(), "contention must prevent clean");
14136 assert!(!next.exists());
14137 assert!(runner.state.handovers.is_empty());
14138 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14140 from_run: "earlier".into(),
14141 generation: 1,
14142 mode: crate::recovery::Mode::VerifyRepair,
14143 reasons: Vec::new(),
14144 reviewed_head: None,
14145 rounds: Vec::new(),
14146 open_findings: Vec::new(),
14147 evidence: None,
14148 criteria_inherited: true,
14149 repair: crate::recovery::Repair::Started {
14150 before: head.clone(),
14151 },
14152 });
14153 assert!(
14154 runner
14155 .recovery_repair(&winner)
14156 .await
14157 .unwrap_err()
14158 .is::<crate::repo_lock::ResourceBlocked>()
14159 );
14160 assert!(matches!(
14161 runner.state.recovery.as_ref().unwrap().repair,
14162 crate::recovery::Repair::Started { .. }
14163 ));
14164 assert!(leftover.exists());
14165 drop(external);
14166 runner.recovery_repair(&winner).await.unwrap();
14167 assert!(!leftover.exists());
14168 assert!(matches!(
14169 runner.state.recovery.as_ref().unwrap().repair,
14170 crate::recovery::Repair::Done { .. }
14171 ));
14172 std::fs::remove_file(&lock_path).unwrap();
14174 std::fs::create_dir(&lock_path).unwrap();
14175 std::fs::write(&leftover, "untrusted edits").unwrap();
14176 let (_, _, out) = runner
14177 .ask_fixer_among(
14178 attempts,
14179 "graft",
14180 None,
14181 Some((&winner.worktree, &head)),
14182 build,
14183 )
14184 .await;
14185 assert!(matches!(out, AgentOutcome::Failed(_)));
14186 assert!(leftover.exists());
14187 assert!(!next.exists());
14188 assert_eq!(
14189 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14190 head
14191 );
14192 }
14193
14194 #[cfg(unix)]
14195 async fn registered_runner(root: &Path) -> Runner {
14196 repository_runner(root, crate::config::test_coordination()).await
14197 }
14198
14199 async fn repository_runner(
14200 root: &Path,
14201 coordination: Option<crate::config::Coordination>,
14202 ) -> Runner {
14203 ask_test_home();
14204 let repo = root.join("repo");
14205 std::fs::create_dir_all(&repo).unwrap();
14206 init_repo(&repo);
14207 if coordination.is_some() {
14208 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14209 }
14210 let mut runner = runner_at(RunStatus::Prep);
14211 runner.state.repo = repo.clone();
14212 runner.state.config.coordination = coordination;
14213 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14214 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14215 runner.state.config.graph.advise = false;
14216 runner.roles.implementers = vec![spec("implement")];
14217 runner.roles.judges = vec![spec("judge")];
14218 runner.prep().await.unwrap();
14219 runner
14220 }
14221
14222 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14223
14224 #[tokio::test]
14225 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14226 let tmp = tempfile::tempdir().unwrap();
14227 let mut runner = repository_runner(tmp.path(), None).await;
14228 let prompts = runner.state.config.prompts.clone();
14229 let run = runner.state.id.clone();
14230 let ctx = WaveCtx {
14231 carry_seats: false,
14232 pauses: None,
14233 run: &run,
14234 node: "advise",
14235 prompts: &prompts,
14236 cache: None,
14237 round: None,
14238 };
14239 let jobs: Vec<_> = ["alpha", "beta"]
14240 .into_iter()
14241 .map(|id| SeatJob {
14242 spec: scripted(id, AUTH_OUTAGE),
14243 seat: SeatState::new(id, id, 7),
14244 prompt: "propose".into(),
14245 cwd: runner.state.candidates[0].worktree.clone(),
14246 timeout: Duration::from_secs(10),
14247 allow_write: false,
14248 sessions: false,
14249 artifacts: agent::artifacts_dir(&runner.state.dir()),
14250 stem: id.into(),
14251 handover: None,
14252 })
14253 .collect();
14254 let mut losses = Vec::new();
14255 let results = ask_wave_with(
14256 jobs.clone(),
14257 runner.sem.clone(),
14258 2,
14259 &[],
14260 &ctx,
14261 &mut losses,
14262 &mut runner.state,
14263 &|_: &str| Ok(()),
14264 )
14265 .await;
14266 assert!(
14267 results
14268 .iter()
14269 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14270 );
14271 assert_eq!(runner.state.quota.len(), 2);
14272 assert!(
14273 runner
14274 .state
14275 .quota
14276 .iter()
14277 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14278 );
14279 assert!(!runner.state.advise_attempted);
14280 runner.stop_for_resource().unwrap();
14281 assert_eq!(runner.state.status, RunStatus::Blocked);
14282 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14283 assert_eq!(
14284 runner.state.resource_block.as_ref().unwrap().kind,
14285 "agent authentication"
14286 );
14287 runner.state.resource_block = None;
14290 runner.state.quota.clear();
14291 let mut mixed = jobs;
14292 mixed[1].spec.command = vec!["false".into()];
14293 let results = ask_wave_with(
14294 mixed,
14295 runner.sem.clone(),
14296 0,
14297 &[],
14298 &ctx,
14299 &mut losses,
14300 &mut runner.state,
14301 &|_: &str| Ok(()),
14302 )
14303 .await;
14304 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14305 assert!(runner.state.resource_block.is_none());
14306 assert_eq!(losses.len(), 1);
14307 }
14308
14309 #[tokio::test]
14310 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14311 let tmp = tempfile::tempdir().unwrap();
14312 let mut runner = repository_runner(tmp.path(), None).await;
14313 let implementer = scripted("implement", AUTH_OUTAGE);
14314 runner.roles.implementers = vec![implementer.clone()];
14315 runner.state.config.agents = vec![implementer];
14316 runner.implement().await.unwrap();
14317 assert_eq!(runner.state.status, RunStatus::Blocked);
14318 assert_eq!(runner.state.quota.len(), 1);
14319 assert_eq!(
14320 runner.state.quota[0].cause,
14321 agent::QuotaCause::Authentication
14322 );
14323 let candidate = &runner.state.candidates[0];
14324 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14325 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14326 }
14327
14328 #[tokio::test]
14329 #[cfg(unix)]
14330 async fn parked_implementer_persists_completed_work() {
14331 for registered in [false, true] {
14332 let tmp = tempfile::tempdir().unwrap();
14333 let mut runner = registered_runner(tmp.path()).await;
14334 if !registered {
14335 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14336 }
14337 let marker = tmp.path().join("started");
14338 let release = tmp.path().join("release");
14339 let mut implementer = spec("implement");
14340 implementer.command = vec![
14341 "sh".into(),
14342 "-c".into(),
14343 format!(
14344 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14345 marker.display(),
14346 release.display()
14347 ),
14348 ];
14349 runner.roles.implementers = vec![implementer.clone()];
14350 runner.state.config.agents = vec![implementer];
14351 let pause = runner.pause.clone();
14352 let (result, ()) = tokio::join!(runner.implement(), async {
14353 tokio::time::timeout(Duration::from_secs(10), async {
14354 while !marker.exists() {
14355 tokio::time::sleep(Duration::from_millis(10)).await;
14356 }
14357 })
14358 .await
14359 .unwrap();
14360 pause.park();
14361 std::fs::write(&release, "go").unwrap();
14362 });
14363 result.unwrap();
14364 let candidate = &runner.state.candidates[0];
14365 assert!(candidate.commits > 0, "{candidate:?}");
14366 assert!(candidate.summary.contains("saved-result"));
14367 assert_eq!(candidate.files, 1);
14368 let before = candidate.commits;
14369 runner.implement().await.unwrap();
14370 assert_eq!(runner.state.candidates[0].commits, before);
14371 }
14372 }
14373
14374 #[tokio::test]
14375 #[cfg(unix)]
14376 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14377 let tmp = tempfile::tempdir().unwrap();
14378 let mut runner = registered_runner(tmp.path()).await;
14379 let winner = runner.state.candidates[0].clone();
14380 let cache = tmp.path().join("cache");
14381 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14382 runner.state.config.graph.timeout_resource_wait = 0;
14383 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14384 let crate::cache::AcquireOutcome::Acquired(held) =
14385 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14386 else {
14387 panic!("cache must be free")
14388 };
14389 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14390 from_run: "earlier".into(),
14391 generation: 1,
14392 mode: crate::recovery::Mode::VerifyRepair,
14393 reasons: Vec::new(),
14394 reviewed_head: None,
14395 rounds: Vec::new(),
14396 open_findings: Vec::new(),
14397 evidence: None,
14398 criteria_inherited: true,
14399 repair: crate::recovery::Repair::NotStarted,
14400 });
14401 runner.recovery_repair(&winner).await.unwrap();
14402 assert!(matches!(
14403 runner.state.recovery.as_ref().unwrap().repair,
14404 crate::recovery::Repair::WaitingForRestore { .. }
14405 ));
14406 drop(held);
14407 runner.state.resource_block = None;
14408 runner.state.config.graph.timeout_resource_wait = 1;
14409 runner.recovery_repair(&winner).await.unwrap();
14410 assert!(matches!(
14411 runner.state.recovery.as_ref().unwrap().repair,
14412 crate::recovery::Repair::Done { .. }
14413 ));
14414 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14415 }
14416
14417 #[tokio::test]
14418 #[cfg(unix)]
14419 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14420 use std::os::fd::AsRawFd as _;
14421 for graft in [false, true] {
14422 let tmp = tempfile::tempdir().unwrap();
14423 let mut runner = registered_runner(tmp.path()).await;
14424 runner.state.config.graph.timeout_resource_wait = 1;
14425 let winner = runner.state.candidates[0].clone();
14426 let marker = tmp.path().join("fixer-started");
14427 let mut fixer = spec("fixer");
14428 fixer.command = vec![
14429 "sh".into(),
14430 "-c".into(),
14431 format!("echo started >> '{}'; echo done", marker.display()),
14432 ];
14433 runner.roles.fixer = Some(vec![fixer]);
14434 if graft {
14435 let mut loser = winner.clone();
14436 loser.label = 'B';
14437 loser.index = 1;
14438 runner.state.candidates.push(loser);
14439 runner.state.config.graph.graft_runner_up = true;
14440 runner.state.tally = Some(Tally {
14441 first_choice: BTreeMap::new(),
14442 borda: BTreeMap::new(),
14443 winner: winner.label,
14444 rankings: 1,
14445 unanimous_initial: true,
14446 deliberated: false,
14447 changed_votes: 0,
14448 unanimous_final: true,
14449 tie_break: None,
14450 judges: 1,
14451 present: 1,
14452 quorum: 1,
14453 met_quorum: true,
14454 uncontested: None,
14455 });
14456 runner.state.judgements.push(crate::run::Judgement {
14457 judge: 1,
14458 seat: "judge-1".into(),
14459 agent: "judge".into(),
14460 ranking: vec![winner.label, 'B'],
14461 reasons: BTreeMap::new(),
14462 confidence: None,
14463 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14464 order: vec![0, 1],
14465 failed: None,
14466 duration_ms: 0,
14467 });
14468 } else {
14469 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14470 from_run: "earlier".into(),
14471 generation: 1,
14472 mode: crate::recovery::Mode::VerifyRepair,
14473 reasons: Vec::new(),
14474 reviewed_head: None,
14475 rounds: Vec::new(),
14476 open_findings: Vec::new(),
14477 evidence: None,
14478 criteria_inherited: true,
14479 repair: crate::recovery::Repair::NotStarted,
14480 });
14481 }
14482 let reader = crate::repo_lock::SeatLease::for_checkout(
14483 &winner.worktree,
14484 false,
14485 crate::config::test_coordination().as_ref(),
14486 )
14487 .await
14488 .unwrap();
14489 let pause = runner.pause.clone();
14490 let (result, ()) = tokio::join!(
14491 async {
14492 if graft {
14493 runner.graft().await
14494 } else {
14495 runner.recovery_repair(&winner).await
14496 }
14497 },
14498 async {
14499 tokio::time::sleep(Duration::from_millis(100)).await;
14500 pause.park();
14501 }
14502 );
14503 result.unwrap();
14504 assert!(!marker.exists(), "a blocked seat must not spawn");
14505 runner.state = RunState::load(&runner.state.id).unwrap();
14507 if graft {
14508 assert_eq!(
14509 runner.state.graft.as_ref().unwrap().outcome,
14510 GraftOutcome::WaitingForRestore
14511 );
14512 } else {
14513 assert!(matches!(
14514 runner.state.recovery.as_ref().unwrap().repair,
14515 crate::recovery::Repair::WaitingForRestore { .. }
14516 ));
14517 }
14518 drop(reader);
14519 runner.pause = Pause::new();
14520 runner.state.resource_block = None;
14521 let external =
14523 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14524 assert_eq!(
14525 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14526 0
14527 );
14528 let leftover = winner.worktree.join("leftover");
14529 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14530 let result = if graft {
14531 runner.graft().await
14532 } else {
14533 runner.recovery_repair(&winner).await
14534 };
14535 assert!(
14536 result
14537 .unwrap_err()
14538 .is::<crate::repo_lock::ResourceBlocked>()
14539 );
14540 assert!(leftover.exists());
14541 assert!(!marker.exists());
14542 drop(external);
14543 runner.state.resource_block = None;
14544 let result = if graft {
14545 runner.graft().await
14546 } else {
14547 runner.recovery_repair(&winner).await
14548 };
14549 result.unwrap();
14550 assert!(!leftover.exists(), "restore must precede retry");
14551 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14552 if graft {
14553 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14554 } else {
14555 assert!(matches!(
14556 runner.state.recovery.as_ref().unwrap().repair,
14557 crate::recovery::Repair::Done { .. }
14558 ));
14559 }
14560 }
14561 }
14562
14563 #[tokio::test]
14564 #[cfg(unix)]
14565 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14566 use std::os::fd::AsRawFd as _;
14567 let tmp = tempfile::tempdir().unwrap();
14568 let mut runner = registered_runner(tmp.path()).await;
14569 runner.state.config.graph.timeout_resource_wait = 1;
14570 let winner = runner.state.candidates[0].clone();
14571 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14572 runner.state.tally = Some(Tally {
14573 first_choice: BTreeMap::new(),
14574 borda: BTreeMap::new(),
14575 winner: winner.label,
14576 rankings: 0,
14577 unanimous_initial: false,
14578 deliberated: false,
14579 changed_votes: 0,
14580 unanimous_final: false,
14581 tie_break: None,
14582 judges: 0,
14583 present: 0,
14584 quorum: 0,
14585 met_quorum: true,
14586 uncontested: Some("fixture".into()),
14587 });
14588 runner.state.graft = Some(GraftRecord {
14589 pre: pre.clone(),
14590 post: None,
14591 lines: None,
14592 limit: None,
14593 agent: None,
14594 outcome: GraftOutcome::Running,
14595 requested: Vec::new(),
14596 adoption: None,
14597 adoption_error: None,
14598 });
14599 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14600 git::commit_all(&winner.worktree, "unfinished graft")
14601 .await
14602 .unwrap();
14603 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14604 let leftover = winner.worktree.join("leftover");
14605 std::fs::write(&leftover, "untrusted edits").unwrap();
14606 let path = runner.state.repo.join(".git/updater.lock");
14607 let external = std::fs::File::create(&path).unwrap();
14608 assert_eq!(
14609 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14610 0
14611 );
14612 assert!(
14613 runner
14614 .graft()
14615 .await
14616 .unwrap_err()
14617 .is::<crate::repo_lock::ResourceBlocked>()
14618 );
14619 assert!(matches!(
14620 runner.state.graft.as_ref().unwrap().outcome,
14621 GraftOutcome::Running
14622 ));
14623 assert_eq!(
14624 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14625 unfinished
14626 );
14627 assert!(leftover.exists());
14628 drop(external);
14629 std::fs::remove_file(&path).unwrap();
14630 std::fs::create_dir(&path).unwrap();
14631 assert!(runner.graft().await.is_err());
14632 assert!(matches!(
14633 runner.state.graft.as_ref().unwrap().outcome,
14634 GraftOutcome::Running
14635 ));
14636 assert_eq!(
14637 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14638 unfinished
14639 );
14640 assert!(leftover.exists());
14641 std::fs::remove_dir(&path).unwrap();
14642 runner.graft().await.unwrap();
14643 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14644 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14645 assert!(!leftover.exists());
14646 }
14647
14648 #[tokio::test]
14649 #[cfg(unix)]
14650 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14651 use std::os::fd::AsRawFd as _;
14652 let tmp = tempfile::tempdir().unwrap();
14653 let mut runner = registered_runner(tmp.path()).await;
14654 runner.state.prep_complete = Some(false);
14655 runner.state.config.graph.timeout_resource_wait = 1;
14656 runner.state.save().unwrap();
14657 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14658 assert_eq!(
14659 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14660 0
14661 );
14662 runner.execute().await.unwrap();
14663 let disk = RunState::load(&runner.state.id).unwrap();
14664 assert!(disk.driver_exited);
14665 assert_eq!(disk.driver_pid, None);
14666 assert_eq!(runner.state.status, RunStatus::Blocked);
14667 assert_eq!(
14668 runner.state.resource_block.as_ref().unwrap().kind,
14669 "repository coordination"
14670 );
14671 assert!(runner.state.prep_failure.is_none());
14672 assert!(runner.state.seats.is_empty());
14673 runner.pause.park();
14674 tokio::time::timeout(Duration::from_secs(10), runner.execute())
14677 .await
14678 .unwrap()
14679 .unwrap();
14680 assert!(runner.state.parked);
14681 assert!(runner.state.prep_failure.is_none());
14682 let disk = RunState::load(&runner.state.id).unwrap();
14683 assert!(disk.driver_exited);
14684 assert_eq!(disk.driver_pid, None);
14685 }
14686
14687 #[tokio::test]
14688 #[cfg(unix)]
14689 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14690 use std::os::fd::AsRawFd as _;
14691 let tmp = tempfile::tempdir().unwrap();
14692 let mut runner = registered_runner(tmp.path()).await;
14693 runner.state.prep_complete = Some(false);
14694 runner.state.config.graph.timeout_resource_wait = 30;
14695 runner.state.save().unwrap();
14696 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14697 assert_eq!(
14698 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14699 0
14700 );
14701 assert!(
14702 tokio::time::timeout(Duration::from_millis(100), runner.execute())
14703 .await
14704 .is_err()
14705 );
14706 let disk = RunState::load(&runner.state.id).unwrap();
14707 assert!(disk.driver_exited);
14708 assert_eq!(disk.driver_pid, None);
14709 assert!(disk.status.resumable());
14710 assert!(runner.state.driver_exited);
14711 assert_eq!(runner.state.driver_pid, None);
14712 runner.state.save().unwrap();
14713 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14714 }
14715
14716 #[tokio::test]
14717 #[cfg(unix)]
14718 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14719 let tmp = tempfile::tempdir().unwrap();
14720 let mut runner = registered_runner(tmp.path()).await;
14721 let winner = runner.state.candidates[0].clone();
14722 let cache = tmp.path().join("cache");
14723 let started = tmp.path().join("pre-gate-started");
14724 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14725 runner.state.config.verify.gate =
14726 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14727 let home = crate::run::home();
14728 let owner =
14729 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14730 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14731 crate::cache::AcquireOutcome::Acquired(g) => g,
14732 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14733 };
14734 let mut gate = Box::pin(runner.run_gate(&winner));
14735 assert!(
14736 tokio::time::timeout(Duration::from_millis(150), &mut gate)
14737 .await
14738 .is_err()
14739 );
14740 assert!(!started.exists(), "pre-gate must wait for the cache too");
14741 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14745 let editor = crate::repo_lock::SeatLease::checkout_with(
14746 &winner.worktree,
14747 true,
14748 crate::config::test_coordination().as_ref(),
14749 &wait,
14750 )
14751 .await
14752 .unwrap();
14753 drop(editor);
14754 guard.release();
14755 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14756 .await
14757 .unwrap()
14758 .unwrap();
14759 drop(gate);
14760 assert!(outcomes.iter().all(CommandOutcome::ok));
14761 assert!(started.exists());
14762 }
14763
14764 #[tokio::test]
14765 #[cfg(unix)]
14766 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14767 use std::os::fd::AsRawFd as _;
14768
14769 ask_test_home();
14770 let tmp = tempfile::tempdir().unwrap();
14771 let repo = tmp.path().join("repo");
14772 std::fs::create_dir_all(&repo).unwrap();
14773 init_repo(&repo);
14774 let common = repo.join(".git");
14775 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14776 let mut runner = runner_at(RunStatus::Prep);
14777 runner.state.repo = repo.clone();
14778 runner.state.config.coordination = crate::config::test_coordination();
14779 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14780 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14781 runner.state.config.graph.advise = false;
14782 runner.roles.implementers = vec![spec("implement")];
14783 runner.roles.judges = vec![spec("judge")];
14784 runner.prep().await.unwrap();
14785 let winner = runner.state.candidates[0].clone();
14786 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14789 let completed = tmp.path().join("seat-completed");
14790 runner.roles.implementers[0].command = vec![
14791 "sh".into(),
14792 "-c".into(),
14793 format!("touch '{}'; echo completed", completed.display()),
14794 ];
14795 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14796 assert_eq!(
14797 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14798 0
14799 );
14800 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14801 let mut implement = Box::pin(runner.implement());
14802 tokio::select! {
14803 result = &mut implement => panic!("rescue must wait: {result:?}"),
14804 result = tokio::time::timeout(Duration::from_secs(10), async {
14805 while !completed.exists() {
14806 tokio::time::sleep(Duration::from_millis(20)).await;
14807 }
14808 }) => result.unwrap(),
14809 }
14810 assert!(
14811 tokio::time::timeout(Duration::from_millis(200), &mut implement)
14812 .await
14813 .is_err()
14814 );
14815 assert_eq!(
14816 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14817 before
14818 );
14819 assert_eq!(
14820 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14821 0
14822 );
14823 tokio::time::timeout(Duration::from_secs(10), &mut implement)
14824 .await
14825 .unwrap()
14826 .unwrap();
14827 drop(implement);
14828 assert_ne!(
14829 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14830 before
14831 );
14832 assert!(git::is_clean(&winner.worktree).await.unwrap());
14833
14834 let started = tmp.path().join("pre-gate-started");
14835 let proceed = tmp.path().join("pre-gate-proceed");
14836 let checked = tmp.path().join("gate-ran");
14837 runner.state.config.verify.pre_gate = vec![format!(
14838 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14839 started.display(),
14840 proceed.display()
14841 )];
14842 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14843 assert_eq!(
14844 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14845 0
14846 );
14847 let mut gate = Box::pin(runner.run_gate(&winner));
14848 assert!(
14849 tokio::time::timeout(Duration::from_millis(200), &mut gate)
14850 .await
14851 .is_err()
14852 );
14853 assert!(!started.exists());
14854 assert!(!checked.exists());
14855 assert!(git::is_clean(&winner.worktree).await.unwrap());
14856 assert_eq!(
14857 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14858 0
14859 );
14860 tokio::select! {
14861 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14862 result = tokio::time::timeout(Duration::from_secs(10), async {
14863 while !started.exists() {
14864 tokio::time::sleep(Duration::from_millis(20)).await;
14865 }
14866 }) => result.unwrap(),
14867 }
14868 assert_eq!(
14869 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14870 -1,
14871 "mechanical edits must hold the real lock"
14872 );
14873 std::fs::write(proceed, "ready").unwrap();
14874 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14875 .await
14876 .unwrap()
14877 .unwrap();
14878 drop(gate);
14879 assert!(outcomes.iter().all(CommandOutcome::ok));
14880 assert!(checked.exists());
14881 assert!(runner.state.pre_gate_commit.is_some());
14882 assert!(git::is_clean(&winner.worktree).await.unwrap());
14883 assert_eq!(
14884 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14885 0,
14886 "gate completion must release coordination"
14887 );
14888 drop(outsider);
14889 std::fs::remove_file(common.join("updater.lock")).unwrap();
14890 std::fs::create_dir(common.join("updater.lock")).unwrap();
14891 std::fs::remove_file(&started).unwrap();
14892 std::fs::remove_file(&checked).unwrap();
14893 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14894 let error = runner.run_gate(&winner).await.unwrap_err();
14895 assert!(
14896 error
14897 .to_string()
14898 .contains("acquire mandatory repository lock")
14899 );
14900 assert!(
14901 !started.exists(),
14902 "failed acquisition must prevent pre-gate"
14903 );
14904 assert!(!checked.exists(), "failed acquisition must prevent gate");
14905 assert!(git::is_clean(&winner.worktree).await.unwrap());
14906 assert_eq!(
14907 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14908 head
14909 );
14910 }
14911
14912 #[tokio::test]
14913 #[cfg(unix)]
14914 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14915 use std::os::fd::AsRawFd as _;
14916 use std::time::Duration;
14917
14918 ask_test_home();
14919 let tmp = tempfile::tempdir().unwrap();
14920 let repo = tmp.path().join("repo");
14921 std::fs::create_dir_all(&repo).unwrap();
14922 init_repo(&repo);
14923 let common = repo.join(".git");
14924 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14925 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14926 assert_eq!(
14927 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14928 0
14929 );
14930 let mut runner = runner_at(RunStatus::Prep);
14931 runner.state.repo = repo.clone();
14932 runner.state.config.coordination = crate::config::test_coordination();
14933 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14934 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14935 runner.state.config.graph.advise = true;
14936 runner.state.config.graph.advisors = 1;
14937 runner.roles.implementers = vec![spec("implement")];
14938 runner.roles.judges = vec![spec("judge")];
14939 let root = runner.state.worktree_root();
14940 let started = tmp.path().join("setup-started");
14941 let proceed = tmp.path().join("setup-proceed");
14942 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14943 run: Some(format!(
14944 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14945 started.display(),
14946 proceed.display()
14947 )),
14948 ..Default::default()
14949 }];
14950 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14951 let config = git::git(&repo, &["config", "--local", "--list"])
14952 .await
14953 .unwrap();
14954 let mut prep = Box::pin(runner.prep());
14955 assert!(
14956 tokio::time::timeout(Duration::from_millis(200), &mut prep)
14957 .await
14958 .is_err()
14959 );
14960 assert!(
14961 !root.exists(),
14962 "contention must prevent even checkout creation"
14963 );
14964 assert!(!started.exists(), "contention must prevent setup");
14965 assert_eq!(
14966 git::git(&repo, &["branch", "--list"]).await.unwrap(),
14967 branches
14968 );
14969 assert_eq!(
14970 git::git(&repo, &["config", "--local", "--list"])
14971 .await
14972 .unwrap(),
14973 config
14974 );
14975 drop(lock);
14976 tokio::select! {
14977 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14978 result = tokio::time::timeout(Duration::from_secs(10), async {
14979 while !started.exists() {
14980 tokio::time::sleep(Duration::from_millis(20)).await;
14981 }
14982 }) => result.unwrap(),
14983 }
14984 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14985 assert_eq!(
14986 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14987 -1
14988 );
14989 assert!(
14990 tokio::time::timeout(
14991 Duration::from_millis(100),
14992 crate::repo_lock::SeatLease::for_checkout(
14993 &repo,
14994 true,
14995 crate::config::test_coordination().as_ref()
14996 )
14997 )
14998 .await
14999 .is_err(),
15000 "preparation must also exclude local editors"
15001 );
15002 std::fs::write(proceed, "ready").unwrap();
15003 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15004 .await
15005 .unwrap()
15006 .unwrap();
15007 drop(prep);
15008 assert_eq!(runner.state.prep_complete, Some(true));
15009 assert!(root.join("judge-1").exists());
15010 assert!(root.join("advisor-1").exists());
15011 assert_eq!(
15012 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15013 0
15014 );
15015 }
15016
15017 #[tokio::test]
15018 #[cfg(unix)]
15019 async fn preparation_lock_failure_prevents_repository_changes() {
15020 ask_test_home();
15021 let tmp = tempfile::tempdir().unwrap();
15022 let repo = tmp.path().join("repo");
15023 std::fs::create_dir_all(&repo).unwrap();
15024 init_repo(&repo);
15025 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15026 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15029 let mut runner = runner_at(RunStatus::Prep);
15030 runner.state.repo = repo.clone();
15031 runner.state.config.coordination = crate::config::test_coordination();
15032 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15033 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15034 runner.roles.implementers = vec![spec("implement")];
15035 let error = runner.prep().await.unwrap_err();
15036 assert!(
15037 error
15038 .to_string()
15039 .contains("acquire mandatory repository lock")
15040 );
15041 assert!(!runner.state.worktree_root().exists());
15042 assert!(runner.state.candidates.is_empty());
15043 assert!(!repo.join(".git/config.worktree").exists());
15044 assert_eq!(
15045 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15046 "* main"
15047 );
15048 }
15049
15050 #[tokio::test]
15051 async fn preparation_records_creation_intent_before_git_can_fail() {
15052 ask_test_home();
15053 let tmp = tempfile::tempdir().unwrap();
15054 let repo = tmp.path().join("repo");
15055 std::fs::create_dir_all(&repo).unwrap();
15056 init_repo(&repo);
15057 let mut runner = runner_at(RunStatus::Prep);
15058 runner.state.repo = repo.clone();
15059 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15060 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15061 runner.state.config.blind.commit_msg_hook = false;
15062 runner.roles.implementers = vec![spec("one")];
15063 let label = blind::assign_labels(1, runner.state.seed)[0];
15064 let branch = runner.state.branch_for(label);
15065 git::git(&repo, &["branch", &branch]).await.unwrap();
15068 assert!(runner.prep().await.is_err());
15069 let saved = RunState::load(&runner.state.id).unwrap();
15070 assert!(saved.candidates.is_empty());
15071 let event = saved
15072 .events
15073 .iter()
15074 .find(|e| e.node == "prep-intent")
15075 .unwrap();
15076 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15077 assert_eq!(intent["branch"], branch);
15078 assert_eq!(intent["start"], runner.state.base_commit);
15079 assert_eq!(
15080 intent["worktree"],
15081 runner
15082 .state
15083 .worktree_root()
15084 .join(format!("cand-{label}"))
15085 .to_string_lossy()
15086 .as_ref()
15087 );
15088 assert_eq!(intent["picks"], serde_json::json!([]));
15089 }
15090
15091 #[tokio::test]
15092 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15093 ask_test_home();
15094 let tmp = tempfile::tempdir().unwrap();
15095 let repo = tmp.path().join("repo");
15096 std::fs::create_dir_all(&repo).unwrap();
15097 init_repo(&repo);
15098 let mut runner = runner_at(RunStatus::Prep);
15099 runner.state.repo = repo.clone();
15100 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15101 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15102 runner.state.sources.pick_commits.push(refs::Input {
15103 token: "missing".into(),
15104 sha: "0".repeat(40),
15105 provenance: refs::Provenance::ExplicitCli,
15106 });
15107 assert!(runner.prep().await.is_err());
15108 assert!(runner.state.candidates.is_empty());
15109 assert!(!runner.state.worktree_root().exists());
15110 }
15111
15112 #[tokio::test]
15113 async fn prose_and_legacy_inference_never_import_into_candidates() {
15114 ask_test_home();
15115 let tmp = tempfile::tempdir().unwrap();
15116 let repo = tmp.path().join("repo");
15117 std::fs::create_dir_all(&repo).unwrap();
15118 init_repo(&repo);
15119 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15120 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15121 .await
15122 .unwrap();
15123 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15124 git::git(&repo, &["add", "."]).await.unwrap();
15125 git::git(&repo, &["commit", "-m", "historical implementation"])
15126 .await
15127 .unwrap();
15128 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15129 git::git(&repo, &["checkout", "main"]).await.unwrap();
15130 for instruction in [
15131 format!("Example: magi/history/A at {source}"),
15132 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15133 format!("Never import magi/history/A or {source}"),
15134 format!("Conversation answer: I mentioned {source} only as background"),
15135 ] {
15136 let mut runner = runner_at(RunStatus::Prep);
15137 runner.state.repo = repo.clone();
15138 runner.state.base_commit = base.clone();
15139 runner.state.instruction = instruction.clone();
15140 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15141 runner.state.config.blind.commit_msg_hook = false;
15142 runner.state.seeds = vec![refs::Seed {
15143 token: "magi/history/A".into(),
15144 sha: source.clone(),
15145 kind: refs::SeedKind::Unmerged,
15146 branch: true,
15147 detail: String::new(),
15148 }];
15149 runner.roles.implementers = vec![spec("one")];
15150 assert_eq!(seeded_instruction(&runner.state), instruction);
15151 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15153 legacy.as_object_mut().unwrap().remove("sources");
15154 runner.state = serde_json::from_value(legacy).unwrap();
15155 assert_eq!(runner.state.sources, refs::Sources::default());
15156 runner.state.save().unwrap();
15157 runner.state = RunState::load(&runner.state.id).unwrap();
15158 runner.prep().await.unwrap();
15159 for candidate in &runner.state.candidates {
15160 assert_eq!(
15161 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15162 base
15163 );
15164 assert!(!candidate.worktree.join("unwanted").exists());
15165 }
15166 }
15167 }
15168
15169 #[tokio::test]
15170 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15171 ask_test_home();
15172 let tmp = tempfile::tempdir().unwrap();
15173 let repo = tmp.path().join("repo");
15174 std::fs::create_dir_all(&repo).unwrap();
15175 init_repo(&repo);
15176 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15177 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15178 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15179 git::git(&repo, &["add", "."]).await.unwrap();
15180 git::git(&repo, &["commit", "-m", "required seed"])
15181 .await
15182 .unwrap();
15183 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15184 git::git(&repo, &["checkout", "main"]).await.unwrap();
15185 let mut runner = runner_at(RunStatus::Prep);
15186 runner.state.repo = repo.clone();
15187 runner.state.base_commit = base.clone();
15188 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15189 runner.state.config.graph.advise = false;
15190 runner.state.config.blind.commit_msg_hook = false;
15191 runner.state.sources =
15192 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15193 .await
15194 .unwrap();
15195 runner.roles.implementers = vec![spec("one"), spec("two")];
15196 runner.roles.judges = vec![spec("judge")];
15197 let label = blind::assign_labels(2, runner.state.seed)[0];
15198 let branch = runner.state.branch_for(label);
15199 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15200 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15201 .await
15202 .unwrap();
15203 runner.state.candidates.push(Candidate {
15204 index: 0,
15205 label,
15206 agent: "one".into(),
15207 branch,
15208 worktree,
15209 summary: String::new(),
15210 stat: String::new(),
15211 files: 0,
15212 commits: 0,
15213 empty: false,
15214 failed: None,
15215 verified_noop: None,
15216 duration_ms: 0,
15217 folded: false,
15218 });
15219 runner.state.save().unwrap();
15220 runner.state = RunState::load(&runner.state.id).unwrap();
15222 assert_eq!(runner.state.prep_complete, Some(false));
15223 runner.prep().await.unwrap();
15224 assert_eq!(runner.state.prep_complete, Some(true));
15225 assert_eq!(runner.state.candidates.len(), 2);
15226 for candidate in &runner.state.candidates {
15227 assert_eq!(
15228 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15229 git::tree_of(&repo, &seed).await.unwrap()
15230 );
15231 assert_eq!(
15232 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15233 "required commit"
15234 );
15235 }
15236 assert!(runner.state.worktree_root().join("judge-1").exists());
15237 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15238 runner.prep().await.unwrap();
15239 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15240
15241 runner.state.prep_complete = Some(false);
15242 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15243 run: Some(format!(
15244 "test -f '{}'",
15245 tmp.path().join("dependency-ready").display()
15246 )),
15247 ..Default::default()
15248 }];
15249 assert!(runner.prep().await.is_err());
15250 assert!(runner.state.prep_failure.is_some());
15251 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15252 runner.state = RunState::load(&runner.state.id).unwrap();
15253 assert!(
15254 runner.prep().await.is_err(),
15255 "correcting the cause alone must not auto-retry"
15256 );
15257 let failed_at = runner.state.events.last().unwrap().at;
15258 assert!(
15259 runner.retry_preparation(failed_at).is_err(),
15260 "old approvals cannot unlock the hold"
15261 );
15262 let mut task = crate::queue::Task::new(
15263 "retry".into(),
15264 "retry setup".into(),
15265 repo.clone(),
15266 crate::queue::Source::Human,
15267 );
15268 task.start(runner.state.id.clone());
15269 task.hold_machine(Some("setup failed".into()));
15270 task.release_operator("cli", None);
15271 runner
15272 .retry_preparation(task.resumption.as_ref().unwrap().at)
15273 .unwrap();
15274 runner.prep().await.unwrap();
15275 assert!(runner.state.prep_failure.is_none());
15276 assert_eq!(runner.state.prep_complete, Some(true));
15277 assert_eq!(runner.state.candidates.len(), 2);
15278 assert!(runner.state.worktree_root().join("judge-1").exists());
15279 for candidate in &runner.state.candidates {
15280 assert_eq!(
15281 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15282 git::tree_of(&repo, &seed).await.unwrap()
15283 );
15284 }
15285 }
15286
15287 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15288 let mut heads = Vec::new();
15289 for candidate in candidates {
15290 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15291 }
15292 heads
15293 }
15294
15295 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15296 ask_test_home();
15297 let tmp = tempfile::tempdir().unwrap();
15298 let repo = tmp.path().join("repo");
15299 std::fs::create_dir(&repo).unwrap();
15300 init_repo(&repo);
15301 let mut runner = runner_at(RunStatus::Reviewing);
15302 runner.state.repo = repo.clone();
15303 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15304 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15305 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15306 let branch = format!("candidate-{label}");
15307 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15308 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15309 .await
15310 .unwrap();
15311 if label == 'B' {
15312 std::fs::write(worktree.join("change"), "winner").unwrap();
15313 git::rescue_commit(&worktree, "candidate change")
15314 .await
15315 .unwrap();
15316 }
15317 runner.state.candidates.push(Candidate {
15318 index,
15319 label,
15320 agent: "alpha".into(),
15321 branch,
15322 worktree,
15323 summary: "completed reply".into(),
15324 stat: String::new(),
15325 files: usize::from(label == 'B'),
15326 commits: usize::from(label == 'B'),
15327 empty: label == 'A',
15328 failed: None,
15329 verified_noop: None,
15330 duration_ms: 1,
15331 folded: false,
15332 });
15333 }
15334 runner.state.judge_skipped = true;
15335 runner.state.tally = Some(Tally {
15336 first_choice: BTreeMap::new(),
15337 borda: BTreeMap::new(),
15338 winner: 'B',
15339 rankings: 0,
15340 unanimous_initial: false,
15341 deliberated: false,
15342 changed_votes: 0,
15343 unanimous_final: false,
15344 tie_break: None,
15345 judges: 0,
15346 present: 0,
15347 quorum: 0,
15348 met_quorum: true,
15349 uncontested: Some("only B produced a change".into()),
15350 });
15351 runner.state.save().unwrap();
15352 (tmp, runner)
15353 }
15354
15355 #[tokio::test]
15356 #[cfg(unix)]
15357 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15358 for (solo, failed_checkpoint) in
15359 [(true, false), (false, false), (true, true), (false, true)]
15360 {
15361 let (_tmp, mut runner) = preservation_resume_fixture().await;
15362 if solo {
15363 runner.state.candidates.retain(|c| c.label == 'A');
15364 }
15365 runner.state.candidates[0].empty = false;
15366 runner.state.candidates[0].summary.clear();
15367 runner.state.tally = None;
15368 runner.state.judge_skipped = false;
15369 runner.state.status = RunStatus::Implementing;
15370 runner.state.config.disk.min_free_bytes = u64::MAX;
15371 let mut implementer = spec("alpha");
15372 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15373 runner.roles.implementers = vec![implementer];
15374 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15375 if failed_checkpoint {
15376 std::fs::create_dir_all(&patch).unwrap();
15377 }
15378 runner.state.save().unwrap();
15379 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15380 runner.state = RunState::load(&runner.state.id).unwrap();
15381 assert_eq!(runner.state.status, RunStatus::Blocked);
15382 assert!(runner.state.interrupted_implementations.contains(&'A'));
15383 assert_eq!(runner.state.candidates[0].commits, 0);
15384 assert!(!runner.state.candidates[0].empty);
15385 assert!(runner.state.candidates[0].failed.is_none());
15386 if failed_checkpoint {
15387 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15388 std::fs::remove_dir(patch).unwrap();
15389 }
15390 runner.state.resource_block = None;
15391 runner.state.config.disk.min_free_bytes = 0;
15392 runner.roles.implementers[0].command = vec![
15393 "sh".into(),
15394 "-c".into(),
15395 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15396 ];
15397 runner.implement().await.unwrap();
15398 assert!(runner.state.interrupted_implementations.is_empty());
15399 assert!(runner.state.candidate_preservation.is_empty());
15400 assert_eq!(runner.state.candidates[0].commits, 1);
15401 assert!(!runner.state.candidates[0].empty);
15402 assert!(
15403 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15404 .unwrap()
15405 .contains("retried empty interrupted seat")
15406 );
15407 if !solo {
15408 assert_eq!(runner.state.candidates[1].commits, 1);
15409 }
15410 }
15411 }
15412
15413 #[tokio::test]
15414 #[cfg(unix)]
15415 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15416 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15417 let (_tmp, mut runner) = preservation_resume_fixture().await;
15418 runner.state.candidates.retain(|c| c.label == 'A');
15419 runner.state.candidates[0].empty = false;
15420 runner.state.candidates[0].summary.clear();
15421 runner.state.tally = None;
15422 runner.state.judge_skipped = false;
15423 runner.state.status = RunStatus::Implementing;
15424 runner.state.config.disk.min_free_bytes = u64::MAX;
15427 let wt = runner.state.candidates[0].worktree.clone();
15428 std::fs::write(
15429 wt.join("interrupted-source"),
15430 "retain implementation work\n",
15431 )
15432 .unwrap();
15433 if committed {
15434 git::rescue_commit(&wt, "implementation checkpoint")
15435 .await
15436 .unwrap();
15437 }
15438 let lock = git::git(
15439 &wt,
15440 &[
15441 "rev-parse",
15442 "--path-format=absolute",
15443 "--git-path",
15444 "index.lock",
15445 ],
15446 )
15447 .await
15448 .unwrap();
15449 if block_rescue {
15450 std::fs::write(&lock, "fixture-owned lock").unwrap();
15451 }
15452 let mut implementer = spec("alpha");
15453 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15454 runner.roles.implementers = vec![implementer];
15455 runner.state.save().unwrap();
15456 let result = runner.implement().await;
15457 assert_eq!(result.is_err(), block_rescue);
15458 assert_eq!(
15459 runner.state.resource_block.as_ref().unwrap().kind,
15460 "disk pressure"
15461 );
15462 assert!(
15463 runner
15464 .state
15465 .events
15466 .iter()
15467 .any(|e| e.message.contains("agent execution interrupted"))
15468 );
15469 assert!(
15470 runner
15471 .state
15472 .dir()
15473 .join("artifacts/candidate-A-recovery/staged.patch")
15474 .exists()
15475 );
15476 if block_rescue {
15477 let saved = RunState::load(&runner.state.id).unwrap();
15478 assert!(saved.candidate_preservation.contains_key(&'A'));
15479 assert!(!saved.candidates[0].empty);
15480 assert_eq!(
15481 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15482 "retain implementation work\n"
15483 );
15484 std::fs::remove_file(lock).unwrap();
15485 runner.state = saved;
15486 } else {
15487 assert_eq!(runner.state.status, RunStatus::Blocked);
15488 assert_eq!(runner.state.candidates[0].commits, 1);
15489 }
15490 runner.roles.implementers.clear();
15493 runner.state.config.disk.min_free_bytes = 0;
15494 runner.state.resource_block = None;
15495 runner.implement().await.unwrap();
15496 assert!(runner.state.candidate_preservation.is_empty());
15497 assert_eq!(runner.state.candidates[0].commits, 1);
15498 assert!(!runner.state.candidates[0].empty);
15499 assert!(git::is_clean(&wt).await.unwrap());
15500 assert!(
15501 crate::run::read_artifact(&runner.state, "cand-A.patch")
15502 .unwrap()
15503 .contains("retain implementation work")
15504 );
15505 }
15506 }
15507
15508 #[tokio::test]
15509 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15510 let (_tmp, mut runner) = preservation_resume_fixture().await;
15511 runner.implement().await.unwrap();
15513 assert!(runner.state.candidate_preservation.is_empty());
15514 assert!(
15515 !runner
15516 .state
15517 .dir()
15518 .join("artifacts/candidate-A-recovery")
15519 .exists()
15520 );
15521 runner.fold_losers().await.unwrap();
15522 assert!(!runner.state.candidates[0].worktree.exists());
15523 for schema in [14, crate::run::SCHEMA] {
15524 runner.state.schema = schema;
15525 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15526 runner.state.save().unwrap();
15527 runner.state = RunState::load(&runner.state.id).unwrap();
15528 runner.implement().await.unwrap();
15529 assert!(runner.state.candidate_preservation.is_empty());
15530 assert!(runner.state.candidates[0].folded);
15531 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15532 }
15533 }
15534
15535 #[tokio::test]
15536 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15537 let (_tmp, mut runner) = preservation_resume_fixture().await;
15538 runner.state.candidates.retain(|c| c.label == 'A');
15539 runner.state.tally = None;
15540 runner.state.judge_skipped = false;
15541 let candidate = &mut runner.state.candidates[0];
15542 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15543 candidate.failed = Some("deployment validation unavailable".into());
15544 std::fs::write(
15545 candidate.worktree.join("repair"),
15546 "future launcher policy\n",
15547 )
15548 .unwrap();
15549 runner
15550 .state
15551 .candidate_preservation
15552 .insert('A', "awaiting persistence".into());
15553 runner.persist_candidates().await.unwrap();
15554 let candidate = &runner.state.candidates[0];
15555 assert!(!candidate.empty);
15556 assert_eq!(candidate.commits, 1);
15557 assert_eq!(candidate.files, 1);
15558 assert!(candidate.failed.is_none());
15559 assert!(candidate.summary.contains("EPERM"));
15560 assert!(
15561 crate::run::read_artifact(&runner.state, "cand-A.patch")
15562 .unwrap()
15563 .contains("future launcher policy")
15564 );
15565 assert!(runner.state.candidate_preservation.is_empty());
15566 }
15567
15568 #[tokio::test]
15569 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15570 let (_tmp, mut runner) = preservation_resume_fixture().await;
15571 let wt = runner.state.candidates[0].worktree.clone();
15572 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15573 runner.state.schema = 14;
15574 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15575 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15576 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15577 for _ in 0..2 {
15578 assert!(runner.implement().await.is_err());
15579 assert_eq!(runner.state.status, RunStatus::Blocked);
15580 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15581 assert!(
15582 runner.state.candidates[0].empty,
15583 "old verdict must remain unusable"
15584 );
15585 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15586 assert!(runner.state.judge_skipped);
15587 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15588 assert!(
15589 crate::run::read_artifact(&runner.state, "cand-A.patch")
15590 .unwrap()
15591 .contains("owned edits")
15592 );
15593 assert!(wt.exists());
15594 assert_eq!(
15595 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15596 .await
15597 .unwrap(),
15598 1
15599 );
15600 runner.state = RunState::load(&runner.state.id).unwrap();
15601 }
15602 }
15603
15604 #[tokio::test]
15605 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15606 let (_tmp, mut runner) = preservation_resume_fixture().await;
15607 let wt = runner.state.candidates[0].worktree.clone();
15608 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15609 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15610 let index_lock = PathBuf::from(
15611 git::git(
15612 &wt,
15613 &[
15614 "rev-parse",
15615 "--path-format=absolute",
15616 "--git-path",
15617 "index.lock",
15618 ],
15619 )
15620 .await
15621 .unwrap(),
15622 );
15623 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15624 runner.state.schema = 14;
15625 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15626 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15627 assert!(runner.implement().await.is_err());
15628 assert!(runner.after_implement().is_err());
15629 assert!(runner.fold_losers().await.is_err());
15630 assert_eq!(
15631 std::fs::read_to_string(wt.join("recovered")).unwrap(),
15632 "failed seat edits"
15633 );
15634 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15635 runner.state = RunState::load(&runner.state.id).unwrap();
15636 std::fs::remove_file(index_lock).unwrap();
15637 assert!(runner.implement().await.is_err());
15639 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15640 assert!(runner.state.candidates[0].empty);
15641 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15642 assert!(
15643 crate::run::read_artifact(&runner.state, "cand-A.patch")
15644 .unwrap()
15645 .contains("failed seat edits")
15646 );
15647 assert!(wt.exists());
15648 }
15649
15650 #[tokio::test]
15651 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15652 let (_tmp, mut runner) = preservation_resume_fixture().await;
15653 let repo = runner.state.repo.clone();
15654 let common = git::git(
15655 &repo,
15656 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15657 )
15658 .await
15659 .unwrap();
15660 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15661 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15662 assert!(runner.fold_losers().await.is_err());
15663 assert!(!runner.state.candidates[0].worktree.exists());
15664 assert!(!runner.state.candidates[0].folded);
15665 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15666 runner.state = RunState::load(&runner.state.id).unwrap();
15667 std::fs::remove_file(branch_lock).unwrap();
15668 let judge = runner.state.worktree_root().join("judge-1");
15669 git::worktree_add_detached(&repo, &judge, "main")
15670 .await
15671 .unwrap();
15672 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15673 runner.roles.judges = vec![spec("judge")];
15674 assert!(runner.fold_losers().await.is_err());
15675 runner.state = RunState::load(&runner.state.id).unwrap();
15676 assert!(
15677 runner.state.candidates[0].folded,
15678 "candidate progress saved before judge failure"
15679 );
15680 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15681 assert!(judge.join("uncommitted").exists());
15682 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15683 runner.fold_losers().await.unwrap();
15684 assert!(!judge.exists());
15685 runner.fold_losers().await.unwrap();
15686 }
15687
15688 #[tokio::test]
15689 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15690 let (_tmp, mut runner) = preservation_resume_fixture().await;
15691 runner.state.candidates.retain(|c| c.label == 'B');
15692 runner.state.tally = None;
15693 runner.state.judge_skipped = false;
15694 runner.state.prep_complete = Some(true);
15695 runner.state.config.graph.advise = false;
15696 runner.state.config.graph.selected_implementer = Some("alpha".into());
15697 let wt = runner.state.candidates[0].worktree.clone();
15698 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15699 runner
15700 .state
15701 .candidate_preservation
15702 .insert('B', "awaiting persistence".into());
15703 let index_lock = git::git(
15704 &wt,
15705 &[
15706 "rev-parse",
15707 "--path-format=absolute",
15708 "--git-path",
15709 "index.lock",
15710 ],
15711 )
15712 .await
15713 .unwrap();
15714 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15715 runner.state.save().unwrap();
15716 assert!(runner.execute_graph().await.is_err());
15717 assert_eq!(runner.state.status, RunStatus::Blocked);
15718 assert!(runner.state.candidate_preservation.contains_key(&'B'));
15719 std::fs::remove_file(index_lock).unwrap();
15720 for _ in 0..2 {
15723 runner.state = RunState::load(&runner.state.id).unwrap();
15724 runner.pause.park();
15725 runner.execute_graph().await.unwrap();
15726 assert_eq!(runner.state.status, RunStatus::Implementing);
15727 assert!(runner.state.candidate_preservation.is_empty());
15728 assert_eq!(runner.state.candidates[0].commits, 2);
15729 assert!(runner.state.tally.is_none());
15730 }
15731 runner.implement().await.unwrap();
15732 assert_eq!(runner.state.status, RunStatus::Judging);
15733 runner.state.status = RunStatus::Blocked;
15735 runner.state.candidates[0].failed = Some("implementation failed".into());
15736 runner.state.save().unwrap();
15737 let error = runner.execute_graph().await.unwrap_err();
15738 assert!(
15739 error
15740 .to_string()
15741 .contains("selected implementation is held")
15742 );
15743 }
15744
15745 #[tokio::test]
15746 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15747 let (_tmp, mut runner) = preservation_resume_fixture().await;
15748 let repo = runner.state.repo.clone();
15749 let candidate = runner.state.candidates[0].clone();
15750 let registered = git::worktree_holding(&repo, &candidate.branch)
15753 .await
15754 .unwrap()
15755 .unwrap();
15756 assert_eq!(
15757 registered.canonicalize().unwrap(),
15758 candidate.worktree.canonicalize().unwrap()
15759 );
15760 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15761 assert_eq!(
15762 git::worktree_holding(&repo, &candidate.branch)
15763 .await
15764 .unwrap(),
15765 Some(registered)
15766 );
15767 assert!(
15768 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15769 "the stale registration initially keeps the branch checked out"
15770 );
15771 runner.fold_losers().await.unwrap();
15772 assert!(runner.state.candidates[0].folded);
15773 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15774 assert_eq!(
15775 git::worktree_holding(&repo, &candidate.branch)
15776 .await
15777 .unwrap(),
15778 None
15779 );
15780 runner.state = RunState::load(&runner.state.id).unwrap();
15781 runner.fold_losers().await.unwrap();
15782 }
15783
15784 #[tokio::test]
15785 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15786 let (_tmp, mut runner) = preservation_resume_fixture().await;
15787 let repo = runner.state.repo.clone();
15788 let candidate = runner.state.candidates[0].clone();
15789 let winner = runner.state.candidates[1].clone();
15790 let registered = git::worktree_holding(&repo, &candidate.branch)
15791 .await
15792 .unwrap()
15793 .unwrap();
15794 assert_eq!(
15795 registered.canonicalize().unwrap(),
15796 candidate.worktree.canonicalize().unwrap()
15797 );
15798 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15799 assert_eq!(
15800 git::worktree_holding(&repo, &candidate.branch)
15801 .await
15802 .unwrap(),
15803 Some(registered)
15804 );
15805 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15806
15807 for _ in 0..2 {
15808 fold_run(&mut runner.state, false, &crate::run::home())
15809 .await
15810 .unwrap();
15811 runner.state = RunState::load(&runner.state.id).unwrap();
15812 assert!(runner.state.candidates[0].folded);
15813 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15814 assert_eq!(
15815 git::worktree_holding(&repo, &candidate.branch)
15816 .await
15817 .unwrap(),
15818 None
15819 );
15820 assert!(!runner.state.candidates[1].folded);
15821 assert!(winner.worktree.exists());
15822 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15823 }
15824 }
15825
15826 #[tokio::test]
15827 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15828 ask_test_home();
15829 let tmp = tempfile::tempdir().unwrap();
15830 let repo = tmp.path().join("repo");
15831 std::fs::create_dir(&repo).unwrap();
15832 init_repo(&repo);
15833 let mut runner = runner_at(RunStatus::Implementing);
15834 runner.state.repo = repo.clone();
15835 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15836 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15837 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15838 let branch = format!("candidate-{label}");
15839 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15840 git::worktree_add_branch(&repo, &wt, &branch, "main")
15841 .await
15842 .unwrap();
15843 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15844 git::git(&wt, &["add", "README.md"]).await.unwrap();
15845 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15846 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15847 runner.state.candidates.push(Candidate {
15848 index,
15849 label,
15850 agent: "alpha".to_owned(),
15851 branch,
15852 worktree: wt,
15853 summary: "completed implementation".to_owned(),
15854 stat: String::new(),
15855 files: 0,
15856 commits: 0,
15857 empty: false,
15858 failed: None,
15859 verified_noop: None,
15860 duration_ms: 42,
15861 folded: false,
15862 });
15863 runner
15864 .state
15865 .candidate_preservation
15866 .insert(label, "awaiting persistence".to_owned());
15867 }
15868 let wt = runner.state.candidates[0].worktree.clone();
15869 let index_lock = git::git(
15870 &wt,
15871 &[
15872 "rev-parse",
15873 "--path-format=absolute",
15874 "--git-path",
15875 "index.lock",
15876 ],
15877 )
15878 .await
15879 .unwrap();
15880 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15881 runner.state.save().unwrap();
15882 assert!(runner.persist_candidates().await.is_err());
15883 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15884 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15885 assert_eq!(
15886 runner.state.candidates[1].commits, 1,
15887 "other results are still collected"
15888 );
15889 assert!(
15890 !runner.state.candidates[0].empty,
15891 "failed rescue is not no change"
15892 );
15893 assert_eq!(
15894 runner.state.candidates[0].summary,
15895 "completed implementation"
15896 );
15897 assert!(runner.after_implement().is_err());
15898 let home = tmp.path().join("fold-home");
15899 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15900 assert!(wt.join("new-file").exists());
15901 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15902 assert!(recovery.join("staged.patch").exists());
15903 assert!(recovery.join("unstaged.patch").exists());
15904 assert_eq!(
15905 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15906 "untracked edits\n"
15907 );
15908 runner.state.begin_pass();
15909 runner.state.end_pass_parked(ParkCause::Upgrade {
15910 from: "1.0".into(),
15911 to: Some("1.1".into()),
15912 });
15913 runner.state.save().unwrap();
15914 runner.state = RunState::load(&runner.state.id).unwrap();
15915 assert!(matches!(
15916 runner.state.passes.last().unwrap().end,
15917 Some(crate::run::PassEnd::Parked {
15918 cause: ParkCause::Upgrade { .. }
15919 })
15920 ));
15921 std::fs::remove_file(index_lock).unwrap();
15922 runner.implement().await.unwrap();
15924 assert!(runner.state.candidate_preservation.is_empty());
15925 assert!(
15926 runner
15927 .state
15928 .candidates
15929 .iter()
15930 .all(|c| c.commits == 1 && !c.empty)
15931 );
15932 assert!(git::is_clean(&wt).await.unwrap());
15933 assert!(
15934 crate::run::read_artifact(&runner.state, "cand-A.patch")
15935 .unwrap()
15936 .contains("later tracked edits")
15937 );
15938 }
15939
15940 #[tokio::test]
15941 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15942 ask_test_home();
15943 let tmp = tempfile::tempdir().unwrap();
15944 let repo = tmp.path().join("repo");
15945 std::fs::create_dir(&repo).unwrap();
15946 init_repo(&repo);
15947 let wt = tmp.path().join("candidate");
15948 git::worktree_add_branch(&repo, &wt, "candidate", "main")
15949 .await
15950 .unwrap();
15951 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15952 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15953 assert!(wt.join("uncommitted").exists());
15954 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15955 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15956 let mut runner = runner_at(RunStatus::Implementing);
15957 runner.state.repo = repo.clone();
15958 runner.state.base_commit = "missing-base".to_owned();
15959 runner.state.candidates.push(Candidate {
15960 index: 0,
15961 label: 'A',
15962 agent: "alpha".into(),
15963 branch: "candidate".into(),
15964 worktree: wt.clone(),
15965 summary: "completed".into(),
15966 stat: String::new(),
15967 files: 0,
15968 commits: 0,
15969 empty: false,
15970 failed: None,
15971 verified_noop: None,
15972 duration_ms: 1,
15973 folded: false,
15974 });
15975 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15976 assert!(
15977 fold_run(&mut runner.state, true, &tmp.path().join("home"))
15978 .await
15979 .is_err()
15980 );
15981 assert!(!runner.state.candidates[0].folded);
15982 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15983 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15984 runner
15985 .state
15986 .candidate_preservation
15987 .insert('A', "pending".into());
15988 assert!(runner.persist_candidates().await.is_err());
15989 assert!(!runner.state.candidates[0].empty);
15990 assert!(!runner.state.candidates[0].folded);
15991 assert!(wt.exists());
15992 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15993 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15994 std::fs::create_dir_all(&patch_path).unwrap();
15995 assert!(
15996 runner.persist_candidates().await.is_err(),
15997 "artifact failure stays pending"
15998 );
15999 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16000 assert!(!runner.state.candidates[0].empty);
16001 std::fs::remove_dir(patch_path).unwrap();
16002 runner.persist_candidates().await.unwrap();
16003 assert!(
16004 runner.state.candidates[0].empty,
16005 "only successful inspection confirms no change"
16006 );
16007 }
16008
16009 fn init_repo(dir: &Path) {
16010 let run = |args: &[&str]| {
16011 let out = std::process::Command::new("git")
16012 .args(args)
16013 .current_dir(dir)
16014 .quiet()
16015 .output()
16016 .expect("spawn git");
16017 assert!(
16018 out.status.success(),
16019 "git {args:?} failed: {}",
16020 String::from_utf8_lossy(&out.stderr)
16021 );
16022 };
16023 run(&["init", "-b", "main"]);
16024 run(&["config", "user.name", "magi test"]);
16025 run(&["config", "user.email", "magi@example.com"]);
16026 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16027 run(&["add", "-A"]);
16028 run(&["commit", "-m", "init"]);
16029 }
16030
16031 fn ask_test_home() {
16039 crate::run::pin_test_home();
16040 }
16041
16042 #[test]
16043 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16044 ask_test_home();
16045 let mut runner = runner_at(RunStatus::Prep);
16046 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16047 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16050 let error = guard
16051 .finish(
16052 &mut runner.state,
16053 Err(
16054 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16055 .into(),
16056 ),
16057 )
16058 .unwrap_err();
16059 assert_eq!(
16060 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16061 std::io::ErrorKind::PermissionDenied
16062 );
16063 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16064 let message = format!("{error:#}");
16065 assert!(message.contains("original refusal"));
16066 assert!(!message.contains("could not save terminal"));
16067 assert_eq!(
16068 error.chain().count(),
16069 2,
16070 "only startup attribution wraps the original error"
16071 );
16072 let saved = RunState::load(&runner.state.id).unwrap();
16073 assert_eq!(saved.status, RunStatus::Blocked);
16074 assert!(!saved.startup.unwrap().complete);
16075 }
16076
16077 #[test]
16078 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16079 ask_test_home();
16080 let mut runner = runner_at(RunStatus::Prep);
16081 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16082 let id = runner.state.id.clone();
16083 drop(guard);
16084 let saved = RunState::load(&id).unwrap();
16085 assert_eq!(saved.status, RunStatus::Blocked);
16086 assert!(!saved.startup.as_ref().unwrap().complete);
16087 assert!(
16088 saved
16089 .startup
16090 .unwrap()
16091 .failure
16092 .unwrap()
16093 .contains("interrupted")
16094 );
16095 assert!(Runner::resume(&id).is_err());
16096 }
16097
16098 #[test]
16099 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16100 ask_test_home();
16101 let mut runner = runner_at(RunStatus::Prep);
16102 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16103 let error = guard
16104 .finish(
16105 &mut runner.state,
16106 Err(
16107 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16108 .into(),
16109 ),
16110 )
16111 .unwrap_err();
16112 assert_eq!(
16113 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16114 std::io::ErrorKind::PermissionDenied
16115 );
16116 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16117 let mut task = crate::queue::Task::new(
16118 "review".into(),
16119 "criteria".into(),
16120 runner.state.repo.clone(),
16121 crate::queue::Source::Human,
16122 );
16123 task.record_startup_failure(failure.clone());
16124 task.record_startup_failure(failure.clone());
16125 assert_eq!(task.startup_failures.len(), 1);
16126 assert!(task.runs.is_empty());
16127 assert_eq!(task.attempts, 0);
16128 }
16129
16130 #[cfg(unix)]
16131 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16132 ask_test_home();
16133 let tmp = tempfile::tempdir().unwrap();
16134 let repo = tmp.path().join("repo");
16135 std::fs::create_dir(&repo).unwrap();
16136 init_repo(&repo);
16137 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16138 let origin = tmp.path().join("origin.git");
16139 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16140 .await
16141 .unwrap();
16142 git::git(
16143 &repo,
16144 &["remote", "add", "origin", origin.to_str().unwrap()],
16145 )
16146 .await
16147 .unwrap();
16148 git::git(&repo, &["checkout", "-b", "fix/retained"])
16149 .await
16150 .unwrap();
16151 std::fs::write(repo.join("retained"), "retained change").unwrap();
16152 git::git(&repo, &["add", "."]).await.unwrap();
16153 git::git(&repo, &["commit", "-m", "retained implementation"])
16154 .await
16155 .unwrap();
16156 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16157 git::git(&repo, &["checkout", "main"]).await.unwrap();
16158 std::fs::write(repo.join("base-change"), "new base").unwrap();
16159 git::git(&repo, &["add", "."]).await.unwrap();
16160 git::git(&repo, &["commit", "-m", "base moved"])
16161 .await
16162 .unwrap();
16163 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16164 git::git(&repo, &["checkout", "fix/retained"])
16165 .await
16166 .unwrap();
16167 let mut runner = runner_at(RunStatus::Voting);
16168 runner.state.repo = repo.clone();
16169 runner.state.base_commit = base;
16170 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16171 runner.state.config.disk.min_free_bytes = 0;
16172 runner.state.config.graph.review_rounds = 1;
16173 runner.state.config.verify.e2e = vec!["true".into()];
16174 runner.state.config.verify.gate = vec!["true".into()];
16175 runner.state.candidates = vec![Candidate {
16176 index: 0,
16177 label: 'A',
16178 agent: "fixture".into(),
16179 branch: "fix/retained".into(),
16180 worktree: repo.clone(),
16181 summary: String::new(),
16182 stat: String::new(),
16183 files: 1,
16184 commits: 1,
16185 empty: false,
16186 failed: None,
16187 verified_noop: None,
16188 duration_ms: 0,
16189 folded: false,
16190 }];
16191 runner.state.tally = Some(Tally {
16192 first_choice: BTreeMap::new(),
16193 borda: BTreeMap::new(),
16194 winner: 'A',
16195 rankings: 0,
16196 unanimous_initial: true,
16197 deliberated: false,
16198 changed_votes: 0,
16199 unanimous_final: true,
16200 tie_break: None,
16201 judges: 0,
16202 present: 0,
16203 quorum: 0,
16204 met_quorum: true,
16205 uncontested: Some("review-only".into()),
16206 });
16207 let mut task = crate::queue::Task::new(
16208 "fixture".into(),
16209 "fixture".into(),
16210 repo,
16211 crate::queue::Source::Human,
16212 );
16213 task.link_run(&runner.state.id);
16214 crate::queue::Queue::open().put(&mut task).unwrap();
16215 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16216 runner.state.base_sync = Some(BaseSync {
16217 tip: String::new(),
16218 behind: 1,
16219 attempts: BASE_SYNC_ROUNDS,
16220 conflict: None,
16221 already_in: None,
16222 });
16223 runner.sync_to_base().await.unwrap();
16224 assert_eq!(runner.state.status, RunStatus::Blocked);
16225 (tmp, runner, head, task.id)
16226 }
16227
16228 #[cfg(unix)]
16229 #[tokio::test]
16230 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16231 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16232 if std::env::var_os(CHILD).is_none() {
16233 let home = tempfile::tempdir().unwrap();
16234 let output = std::process::Command::new(std::env::current_exe().unwrap())
16235 .args([
16236 "--exact",
16237 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16238 "--nocapture",
16239 ])
16240 .env(CHILD, "1")
16241 .env("MAGI_HOME", home.path())
16242 .env("MAGI_WORKER_HOME", home.path())
16243 .env("MAGI_WORKER_ID", "abcdef")
16244 .output()
16245 .unwrap();
16246 assert!(
16247 output.status.success()
16248 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16249 "{}\n{}",
16250 String::from_utf8_lossy(&output.stdout),
16251 String::from_utf8_lossy(&output.stderr)
16252 );
16253 return;
16254 }
16255 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16256 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16257 let home = crate::run::home();
16258 let queue = crate::queue::Queue::open();
16259 let mut owner = queue.get(&task).unwrap();
16260 owner.release_operator("test-release", None);
16261 owner.start(runner.state.id.clone());
16262 queue.put(&mut owner).unwrap();
16263 runner.state.driver_exited = true;
16264 runner.state.task_instruction = Some(owner.instruction.clone());
16265 runner.state.save().unwrap();
16266 let _claim = queue.claim(&task).unwrap();
16267 let reserve = |id: &str| {
16268 let directory = home.join("workers/jobs").join(id);
16269 std::fs::create_dir_all(&directory).unwrap();
16270 let job = crate::worker::Job {
16271 id: id.into(),
16272 sequence: 1,
16273 kind: "task".into(),
16274 key: format!("task:{task}"),
16275 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16276 generation: "fixture-generation".into(),
16277 version: None,
16278 status: crate::worker::Status::Running,
16279 result: None,
16280 pid: Some(std::process::id()),
16281 created_at: 0,
16282 finished_at: None,
16283 };
16284 std::fs::write(
16285 directory.join("job.json"),
16286 serde_json::to_vec(&job).unwrap(),
16287 )
16288 .unwrap();
16289 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16290 };
16291 let _own = reserve("abcdef");
16292 let other = reserve("fedcba");
16293 assert!(
16294 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16295 .await
16296 .is_err()
16297 );
16298 assert!(
16299 RunState::load(&runner.state.id)
16300 .unwrap()
16301 .base_sync_recovery
16302 .authorizations
16303 .is_empty()
16304 );
16305 drop(other);
16306 assert!(crate::daemon::is_working_on(
16307 &home,
16308 &runner.state.id,
16309 Timestamp::now()
16310 ));
16311 assert!(
16312 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16313 );
16314 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16315 .await
16316 .unwrap();
16317 assert!(
16318 RunState::load(&runner.state.id)
16319 .unwrap()
16320 .base_recovery_pending()
16321 );
16322 }
16323
16324 #[cfg(unix)]
16325 #[tokio::test]
16326 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16327 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16328 let home = crate::run::home();
16329 runner.state.driver_pid = Some(std::process::id());
16330 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16331 runner.state.driver_exited = false;
16332 runner.state.begin_pass();
16333 runner.state.save().unwrap();
16334 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16335 driver.record(&runner.state).unwrap();
16336 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16337 drop(driver);
16339 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16340 assert_eq!(
16341 runner.state.liveness_under(&home, false),
16342 crate::run::Liveness::Dead
16343 );
16344 let queue = crate::queue::Queue::open();
16345 let _claim = queue.claim(&task).unwrap();
16346 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16347 .await
16348 .unwrap();
16349 let disk = RunState::load(&runner.state.id).unwrap();
16350 assert!(!disk.driver_exited);
16351 assert!(disk.base_recovery_pending());
16352 }
16353
16354 #[cfg(unix)]
16355 #[tokio::test]
16356 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16357 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16358 if std::env::var_os(CHILD).is_none() {
16359 use std::os::unix::fs::PermissionsExt;
16360 let tmp = tempfile::tempdir().unwrap();
16361 let gh = tmp.path().join("gh");
16362 std::fs::write(&gh, r#"#!/bin/sh
16363case "$1 $2" in
16364 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16365 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16366 *) exit 2 ;;
16367esac
16368"#).unwrap();
16369 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16370 let mut paths = vec![tmp.path().to_path_buf()];
16371 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16372 let output = std::process::Command::new(std::env::current_exe().unwrap())
16373 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16374 .env(CHILD, "1")
16375 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16376 .env("PATH", std::env::join_paths(paths).unwrap())
16377 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16378 .output().unwrap();
16379 assert!(
16380 output.status.success()
16381 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16382 "{}\n{}",
16383 String::from_utf8_lossy(&output.stdout),
16384 String::from_utf8_lossy(&output.stderr)
16385 );
16386 return;
16387 }
16388 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16389 let repo = runner.state.repo.clone();
16390 git::git(&repo, &["checkout", "main"]).await.unwrap();
16391 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16392 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16393 git::git(&repo, &["checkout", "fix/retained"])
16394 .await
16395 .unwrap();
16396 runner.state.pr = Some(crate::run::PrRecord {
16397 url: "https://example.test/project/pull/1".into(),
16398 number: 1,
16399 state: "open".into(),
16400 checks: "unknown".into(),
16401 round: 0,
16402 rounds: 1,
16403 red_at_merge: Vec::new(),
16404 });
16405 runner.sync_to_base().await.unwrap();
16406 assert_eq!(runner.state.status, RunStatus::Blocked);
16407 assert_eq!(
16408 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16409 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16410 );
16411 assert!(runner.state.base_sync_closure_pending());
16412 assert!(crate::handover::can_resume(&runner.state));
16413 let queue = crate::queue::Queue::open();
16414 assert!(!crate::base_sync_recovery::held_task(
16415 &queue,
16416 &queue.get(&task).unwrap()
16417 ));
16418 assert!(runner.state.authorize_base_sync(&task).is_err());
16419 runner.state.base_sync_recovery.stop = None;
16421 runner.state.status = RunStatus::Voting;
16422 runner.state.gate_ran = true;
16423 runner.state.config.graph.review_rounds = 0;
16424 runner.state.save().unwrap();
16425 assert!(crate::handover::can_resume(&runner.state));
16426 runner.state = RunState::load(&runner.state.id).unwrap();
16427 runner.execute_graph().await.unwrap();
16428 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16429 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16430 assert_eq!(
16431 runner.state.base_sync.as_ref().unwrap().attempts,
16432 BASE_SYNC_ROUNDS
16433 );
16434 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16435 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16436 assert!(runner.state.reviews.is_empty());
16437 assert!(runner.state.gate.is_empty());
16438 }
16439
16440 #[cfg(unix)]
16441 #[tokio::test]
16442 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16443 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16444 runner.state.driver_exited = true;
16445 runner.state.save().unwrap();
16446 let home = crate::run::home();
16447 let queue = crate::queue::Queue::open();
16448 let _claim = queue.claim(&task).unwrap();
16449 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16450 assert!(
16451 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16452 .await
16453 .is_err()
16454 );
16455 drop(lease);
16456 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16457 assert!(
16458 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16459 .await
16460 .is_err()
16461 );
16462 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16463 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16464 .await
16465 .unwrap();
16466 let disk = RunState::load(&runner.state.id).unwrap();
16467 assert!(disk.base_recovery_pending());
16468 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16469 assert!(
16470 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16471 .await
16472 .is_err()
16473 );
16474 assert_eq!(
16475 RunState::load(&runner.state.id)
16476 .unwrap()
16477 .base_sync_recovery
16478 .authorizations
16479 .len(),
16480 1
16481 );
16482 let mut owner = queue.get(&task).unwrap();
16483 owner.runs.push("20260101-000000-ffff".into());
16484 queue.put(&mut owner).unwrap();
16485 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16486 .await
16487 .unwrap_err();
16488 assert!(error.to_string().contains("run task ownership changed"));
16489 }
16490
16491 #[cfg(unix)]
16492 #[tokio::test]
16493 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16494 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16495 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16496 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16497 legacy["status"] = serde_json::json!("voting");
16498 runner.state = serde_json::from_value(legacy).unwrap();
16499 runner.state.save().unwrap();
16500 for _ in 0..3 {
16501 runner.execute_graph().await.unwrap();
16502 assert_eq!(runner.state.status, RunStatus::Blocked);
16503 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16504 assert!(!crate::handover::can_resume(&runner.state));
16505 assert_eq!(
16506 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16507 head
16508 );
16509 assert!(runner.state.reviews.is_empty());
16510 assert!(!runner.state.gate_ran);
16511 }
16512 let notice = crate::notices::run_ended(&runner.state).unwrap();
16513 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16514 runner.state.begin_pass();
16515 runner.state.status = RunStatus::Voting;
16516 runner.deliberate().await.unwrap();
16517 runner.vote().await.unwrap();
16518 runner.finish_after_tally().await.unwrap();
16519 assert_eq!(runner.state.status, RunStatus::Blocked);
16520 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16521 assert_eq!(
16522 crate::notices::run_ended(&runner.state).unwrap().since,
16523 notice.since
16524 );
16525 }
16526
16527 #[cfg(unix)]
16528 #[tokio::test]
16529 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16530 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16531 for detail in [
16532 "fix/retained conflicts with origin/main and did not rebase: conflict",
16533 "rebased fix/retained locally but origin refused the push",
16534 "unknown failure",
16535 ] {
16536 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16537 runner.state.record_base_sync_stop(&head);
16538 assert!(runner.state.authorize_base_sync(&task).is_err());
16539 runner.state.status = RunStatus::Voting;
16540 runner.execute_graph().await.unwrap();
16541 assert_eq!(runner.state.status, RunStatus::Blocked);
16542 assert!(!crate::handover::can_resume(&runner.state));
16543 assert_eq!(
16544 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16545 head
16546 );
16547 }
16548 }
16549
16550 #[cfg(unix)]
16551 #[tokio::test]
16552 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16553 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16554 runner.state.authorize_base_sync(&task).unwrap();
16555 assert!(runner.state.consume_base_recovery(&head));
16556 runner.state.save().unwrap();
16557 runner.state = RunState::load(&runner.state.id).unwrap();
16558 runner.execute_graph().await.unwrap();
16559 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16560 assert_eq!(runner.state.status, RunStatus::Blocked);
16561 assert!(!runner.state.consume_base_recovery(&head));
16562 assert!(!crate::handover::can_resume(&runner.state));
16563 runner.state.authorize_base_sync(&task).unwrap();
16564 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
16565 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16566 assert_eq!(
16567 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16568 head
16569 );
16570 }
16571
16572 #[cfg(unix)]
16573 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
16574 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
16575 runner.state.prep_complete = Some(true);
16577 runner.state.base_sync = None;
16578 runner.state.base_sync_recovery = Default::default();
16579 runner.state.config.graph.gate_fix_rounds = 1;
16580 let mut review = review_round(true, 0, 1, 1, false, true);
16581 review.head = head.clone();
16582 review.verified_head = Some(head.clone());
16583 runner.state.reviews = vec![review];
16584 runner.state.gate = vec![CommandOutcome {
16585 command: "old gate".into(),
16586 code: Some(1),
16587 output_tail: "old failure".into(),
16588 duration_ms: 0,
16589 resource_blocked: false,
16590 }];
16591 runner.state.gate_head = Some(head.clone());
16592 runner.state.gate_ran = true;
16593 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
16594 agent: "fixture".into(),
16595 failed: runner.state.gate.clone(),
16596 notes: "spent".into(),
16597 committed: false,
16598 error: None,
16599 }];
16600 runner.state.land_approval = Some(crate::run::LandApproval {
16601 question: "old approval".into(),
16602 head: head.clone(),
16603 });
16604 runner.state.status = RunStatus::Blocked;
16605 (tmp, runner, head)
16606 }
16607
16608 #[cfg(unix)]
16609 #[tokio::test]
16610 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
16611 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
16612 let (tmp, mut runner, old) = resumed_gate_fixture().await;
16613 let phases = tmp.path().join("phases");
16614 runner.state.config.verify.e2e = vec![format!(
16615 "echo e2e >> '{}'; {}",
16616 phases.display(),
16617 if e2e_ok {
16618 "true"
16619 } else {
16620 "echo new-e2e-failure; false"
16621 }
16622 )];
16623 runner.state.config.verify.gate = vec![format!(
16624 "echo gate >> '{}'; {}",
16625 phases.display(),
16626 if gate_ok {
16627 "true"
16628 } else {
16629 "echo new-gate-failure; false"
16630 }
16631 )];
16632 runner.state.save().unwrap();
16633 runner.state = RunState::load(&runner.state.id).unwrap();
16634 runner.sync_to_base().await.unwrap();
16635 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16636 assert_ne!(head, old);
16637 runner.state = RunState::load(&runner.state.id).unwrap();
16639 runner.execute_graph().await.unwrap();
16640 assert_eq!(runner.state.gate_fixes.len(), 1);
16641 assert_eq!(runner.state.reviews.len(), 1);
16642 assert_eq!(runner.state.reviews[0].head, old);
16643 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
16644 assert_eq!(runner.state.verification_history.len(), 1);
16645 let history = &runner.state.verification_history[0];
16646 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
16647 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16648 assert!(history.archived_at.is_some());
16649 assert_eq!(history.fixes_spent, Some(1));
16650 assert_eq!(history.gate[0].output_tail, "old failure");
16651 let archive_events: Vec<_> = runner
16652 .state
16653 .events
16654 .iter()
16655 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
16656 .collect();
16657 assert_eq!(archive_events.len(), 1);
16658 assert!(archive_events[0].message.contains(&short(&old)));
16659 assert!(archive_events[0].message.contains(&short(&head)));
16660 assert!(
16661 archive_events[0]
16662 .message
16663 .contains("1 gate-fix round(s) already spent stay spent")
16664 );
16665 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
16666 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
16667 if e2e_ok {
16668 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16669 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
16670 } else {
16671 assert!(!runner.state.gate_ran);
16672 }
16673 assert_eq!(
16674 runner.state.status,
16675 if e2e_ok && gate_ok {
16676 RunStatus::Ready
16677 } else {
16678 RunStatus::Blocked
16679 }
16680 );
16681 if !e2e_ok || !gate_ok {
16682 let cause = runner.state.verification_blocking_cause().unwrap();
16683 assert!(cause.contains(&head));
16684 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
16685 runner.state.event("land", "rebased successfully");
16687 let notice = crate::notices::run_ended(&runner.state).unwrap();
16688 assert!(!format!("{notice:?}").contains("rebased successfully"));
16689 assert!(runner.state.merge.is_none());
16690 }
16691 let before = std::fs::read_to_string(&phases).unwrap();
16692 runner.state.save().unwrap();
16693 runner.state = RunState::load(&runner.state.id).unwrap();
16694 runner.review_loop().await.unwrap();
16695 runner.gate().await.unwrap();
16696 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
16697 assert_eq!(runner.state.verification_history.len(), 1);
16698 let history = &runner.state.verification_history[0];
16699 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16700 assert!(history.archived_at.is_some());
16701 assert_eq!(history.fixes_spent, Some(1));
16702 }
16703 }
16704
16705 #[cfg(unix)]
16706 #[tokio::test]
16707 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
16708 let (tmp, mut runner, head) = resumed_gate_fixture().await;
16709 let marker = tmp.path().join("gate-ran");
16710 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
16711 runner.review_loop().await.unwrap();
16712 runner.gate().await.unwrap();
16713 assert_eq!(runner.state.status, RunStatus::Blocked);
16714 assert!(!marker.exists());
16715 assert!(runner.state.verification_history.is_empty());
16716 runner.state.gate_head = None;
16717 runner.gate().await.unwrap();
16718 assert!(marker.exists());
16719 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16720 assert_eq!(runner.state.verification_history.len(), 1);
16721 assert_eq!(runner.state.gate_fixes.len(), 1);
16722 }
16723
16724 #[cfg(unix)]
16725 #[tokio::test]
16726 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
16727 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16728 runner.sync_to_base().await.unwrap();
16729 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16730 runner.state.reconcile_verification_head(&head);
16731 for outcomes in [
16733 Vec::new(),
16734 vec![CommandOutcome {
16735 command: "true".into(),
16736 code: None,
16737 output_tail: "cache unavailable".into(),
16738 duration_ms: 0,
16739 resource_blocked: true,
16740 }],
16741 ] {
16742 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16743 head: head.clone(),
16744 passed: false,
16745 outcomes,
16746 });
16747 runner.state.gate_ran = false;
16748 runner.state.gate.clear();
16749 runner.state.save().unwrap();
16750 runner.state = RunState::load(&runner.state.id).unwrap();
16751 runner.merge().await.unwrap();
16752 assert!(runner.state.merge.is_none());
16753 runner.review_loop().await.unwrap();
16754 runner.gate().await.unwrap();
16755 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
16756 assert!(runner.state.gate_allows_progress());
16757 assert_eq!(runner.state.gate_fixes.len(), 1);
16758 assert!(runner.state.merge.is_none());
16759 }
16760 git::git(
16762 &runner.state.repo,
16763 &["commit", "--allow-empty", "-m", "external update"],
16764 )
16765 .await
16766 .unwrap();
16767 runner.merge().await.unwrap();
16768 assert!(runner.state.merge.is_none());
16769 assert_eq!(runner.state.status, RunStatus::Blocked);
16770 }
16771
16772 #[cfg(unix)]
16773 #[tokio::test]
16774 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16775 for waiting_on_gate in [false, true] {
16776 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16777 runner.sync_to_base().await.unwrap();
16778 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16779 runner.state.reconcile_verification_head(&head);
16780 if waiting_on_gate {
16781 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16782 head: head.clone(),
16783 passed: true,
16784 outcomes: Vec::new(),
16785 });
16786 }
16787 let cache = tmp.path().join("cache");
16788 runner.state.config.verify.e2e =
16789 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16790 runner.state.config.verify.gate = vec!["true".into()];
16791 runner.state.config.graph.timeout_resource_wait = 1;
16792 let owner =
16793 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16794 let guard =
16795 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16796 crate::cache::AcquireOutcome::Acquired(g) => g,
16797 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16798 };
16799 runner.review_loop().await.unwrap();
16800 runner.gate().await.unwrap();
16801 assert!(!runner.state.gate_ran);
16802 assert_eq!(runner.state.gate_fixes.len(), 1);
16803 assert!(runner.state.resource_block.is_some());
16804 runner.merge().await.unwrap();
16805 assert!(runner.state.merge.is_none());
16806 drop(guard);
16807 runner.state = RunState::load(&runner.state.id).unwrap();
16808 runner.state.resource_block = None;
16810 runner.review_loop().await.unwrap();
16811 runner.gate().await.unwrap();
16812 assert!(runner.state.gate_allows_progress());
16813 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16814 assert_eq!(runner.state.gate_fixes.len(), 1);
16815 }
16816 }
16817
16818 #[cfg(unix)]
16819 #[tokio::test]
16820 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16821 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16822 runner.sync_to_base().await.unwrap();
16823 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16824 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16825 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16827 runner.review_loop().await.unwrap();
16828 runner.gate().await.unwrap();
16829 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16830 assert_ne!(before, after);
16831 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16832 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16833 assert!(!runner.state.gate_allows_progress());
16834 runner.merge().await.unwrap();
16835 assert!(runner.state.merge.is_none());
16836 assert!(
16837 runner
16838 .state
16839 .verification_history
16840 .iter()
16841 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16842 );
16843
16844 runner.state.reviews[0].clean = false;
16846 runner.state.reviews[0].answered = 0;
16847 runner.state.config.verify.e2e = vec!["true".into()];
16848 runner.review_loop().await.unwrap();
16849 runner.gate().await.unwrap();
16850 runner.merge().await.unwrap();
16851 assert_eq!(runner.state.status, RunStatus::Blocked);
16852 assert!(runner.state.merge.is_none());
16853 assert!(
16854 runner
16855 .state
16856 .verification_blocking_cause()
16857 .unwrap()
16858 .contains("review blocked")
16859 );
16860 }
16861
16862 #[cfg(unix)]
16863 #[tokio::test]
16864 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16865 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16866 runner.state.gate_head = None;
16867 runner.state.config.verify.gate =
16868 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16869 runner.review_loop().await.unwrap();
16870 runner.gate().await.unwrap();
16871 assert_ne!(
16872 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16873 old
16874 );
16875 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16876 assert!(!runner.state.gate_allows_progress());
16877 assert_eq!(runner.state.status, RunStatus::Blocked);
16878 runner.merge().await.unwrap();
16879 assert!(runner.state.merge.is_none());
16880 }
16881
16882 #[cfg(unix)]
16883 #[tokio::test]
16884 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16885 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16886 let mut old_review = review_round(true, 0, 1, 1, false, true);
16887 old_review.head = head.clone();
16888 old_review.verified_head = Some(head.clone());
16889 runner.state.reviews = vec![old_review];
16890 runner.state.gate = vec![CommandOutcome {
16891 command: "old gate".into(),
16892 code: None,
16893 output_tail: "timed out after 2400s".into(),
16894 duration_ms: 2_400_000,
16895 resource_blocked: false,
16896 }];
16897 runner.state.gate_ran = true;
16898 runner.state.gate_head = Some(head.clone());
16899 runner.state.land_approval = Some(crate::run::LandApproval {
16900 question: "old-head-approval".into(),
16901 head: head.clone(),
16902 });
16903 runner.state.authorize_base_sync(&task).unwrap();
16904 runner.sync_to_base().await.unwrap();
16905 assert!(!runner.state.base_sync_blocked());
16906 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16907 assert_ne!(head, fresh);
16908 assert!(runner.state.reviews.is_empty());
16909 assert!(!runner.state.gate_ran);
16910 let saved = &runner.state.base_sync_recovery.authorizations[0];
16911 assert_eq!(saved.reviews[0].head, head);
16912 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16913 assert!(saved.gate_ran);
16914 assert!(!saved.gate[0].ok());
16915 runner.review_loop().await.unwrap();
16916 runner.gate().await.unwrap();
16917 assert_eq!(runner.state.reviews[0].head, fresh);
16918 assert_eq!(
16919 runner.state.reviews[0].verified_head.as_deref(),
16920 Some(fresh.as_str())
16921 );
16922 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16923 assert!(runner.state.gate_ran);
16924 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16925 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16926 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16927 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16928 assert_eq!(
16929 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16930 "retained change"
16931 );
16932 }
16933
16934 #[cfg(unix)]
16935 #[tokio::test]
16936 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16937 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16938 let queue = crate::queue::Queue::open();
16939 let mut owner = queue.get(&task).unwrap();
16940 owner.release_operator("test-resume", None);
16941 queue.put(&mut owner).unwrap();
16942 let generation = owner.resumption.as_ref().unwrap().generation;
16943 runner.state.task_instruction = Some(owner.instruction.clone());
16944 runner.state.driver_exited = true;
16945 runner.state.save().unwrap();
16946 let home = crate::run::home();
16947 let _claim = queue.claim(&task).unwrap();
16948 owner.resumption.as_mut().unwrap().at =
16949 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16950 queue.put(&mut owner).unwrap();
16951 assert!(
16952 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16953 .await
16954 .is_err(),
16955 "an approval from before the stop grants nothing"
16956 );
16957 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16958 queue.put(&mut owner).unwrap();
16959 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16960 .await
16961 .unwrap();
16962 assert!(
16963 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16964 .await
16965 .is_err()
16966 );
16967 runner.state = RunState::load(&runner.state.id).unwrap();
16968 assert_eq!(
16969 runner.state.base_sync_recovery.authorizations[0].generation,
16970 Some(generation)
16971 );
16972 runner.sync_to_base().await.unwrap();
16973 let repo = runner.state.repo.clone();
16974 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16975 git::git(&repo, &["checkout", "main"]).await.unwrap();
16976 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16977 git::git(&repo, &["add", "."]).await.unwrap();
16978 git::git(&repo, &["commit", "-m", "move base again"])
16979 .await
16980 .unwrap();
16981 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16982 git::git(&repo, &["checkout", "fix/retained"])
16983 .await
16984 .unwrap();
16985 runner.finish_after_tally().await.unwrap();
16986 assert_eq!(runner.state.status, RunStatus::Blocked);
16987 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16988 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16989 assert!(!runner.state.base_recovery_pending());
16990 assert!(!crate::handover::can_resume(&runner.state));
16991 assert!(!runner.state.gate_ran);
16992 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16994 queue.put(&mut owner).unwrap();
16995 assert!(
16996 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16997 .await
16998 .is_err()
16999 );
17000 runner.state = RunState::load(&runner.state.id).unwrap();
17001 runner.sync_to_base().await.unwrap();
17002 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17003 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
17004 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17005 }
17006
17007 #[cfg(unix)]
17008 #[tokio::test]
17009 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
17010 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17011 runner.state.base_sync = None;
17012 runner.state.status = RunStatus::Blocked;
17013 let mut second = runner.state.candidates[0].clone();
17014 second.label = 'B';
17015 runner.state.candidates.push(second);
17016 assert!(runner.state.votes.is_empty());
17017 assert!(runner.state.deliberation.is_empty());
17018 runner.deliberate().await.unwrap();
17019 runner.vote().await.unwrap();
17020 assert_eq!(runner.state.status, RunStatus::Blocked);
17021 assert!(runner.state.votes.is_empty());
17022 assert!(runner.state.deliberation.is_empty());
17023 }
17024
17025 fn runner_at(status: RunStatus) -> Runner {
17028 let mut state = RunState::new(
17029 PathBuf::from("/nonexistent/repo"),
17030 "main".to_owned(),
17031 "deadbeef".to_owned(),
17032 "task".to_owned(),
17033 Config::default(),
17034 );
17035 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
17038 state.id = format!(
17039 "20000101-000000-{:04x}",
17040 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
17041 );
17042 state.status = status;
17043 Runner {
17044 state,
17045 roles: ResolvedRoles {
17046 implementers: Vec::new(),
17047 judges: Vec::new(),
17048 reviewers: Vec::new(),
17049 fixer: None,
17050 conductor: conductor(),
17051 implementer_roster: Vec::new(),
17052 judge_roster: Vec::new(),
17053 reviewer_roster: Vec::new(),
17054 },
17055 sem: Arc::new(Semaphore::new(1)),
17056 pause: Pause::new(),
17057 interrupt: Pause::new(),
17058 }
17059 }
17060
17061 #[tokio::test]
17062 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
17063 crate::run::pin_test_home();
17064 let mut runner = runner_at(RunStatus::Gating);
17065 runner.state.save().unwrap();
17066 let id = runner.state.id.clone();
17067 let passes = runner.state.passes.len();
17068 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
17069 let mut claiming = Box::pin(runner.claim_driver());
17070 std::future::poll_fn(|cx| {
17071 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
17072 std::task::Poll::Ready(())
17073 })
17074 .await;
17075 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
17076 drop(owner);
17077 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
17078 .await
17079 .unwrap()
17080 .unwrap();
17081 assert_eq!(execution.runner.state.passes.len(), passes + 1);
17082 assert!(execution.runner.state.resource_block.is_none());
17083 assert_eq!(
17084 execution.runner.state.liveness(false),
17085 crate::run::Liveness::Live
17086 );
17087 drop(execution);
17088 assert!(RunState::load(&id).unwrap().driver_exited);
17089 }
17090
17091 #[tokio::test]
17092 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
17093 crate::run::pin_test_home();
17094 let mut runner = runner_at(RunStatus::Gating);
17095 runner.state.save().unwrap();
17096 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
17099 assert!(runner.claim_driver().await.is_err());
17100 assert!(runner.state.driver_exited);
17101 assert_eq!(runner.state.driver_pid, None);
17102 let disk = RunState::load(&runner.state.id).unwrap();
17103 assert!(disk.driver_exited);
17104 assert_eq!(disk.driver_pid, None);
17105 assert_eq!(disk.status, RunStatus::Gating);
17106 }
17107
17108 #[tokio::test]
17109 async fn an_early_graph_error_releases_its_driver() {
17110 crate::run::pin_test_home();
17111 let mut runner = runner_at(RunStatus::Blocked);
17112 runner.state.config.graph.selected_implementer = Some("selected".into());
17113 runner.state.save().unwrap();
17114 assert!(runner.execute().await.is_err());
17115 let disk = RunState::load(&runner.state.id).unwrap();
17116 assert!(disk.driver_exited);
17117 assert_eq!(disk.driver_pid, None);
17118 assert_eq!(disk.status, RunStatus::Blocked);
17119 }
17120
17121 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
17122 crate::run::MergeRepair {
17123 question: "repair-question".into(),
17124 head: "old-head".into(),
17125 pr: "https://github.com/o/r/pull/42".into(),
17126 reviews: vec![review_round(false, 1, 1, 1, false, true)],
17127 gate: vec![outcome(Some(0), false)],
17128 gate_head: Some("old-head".into()),
17129 gate_fixes: Vec::new(),
17130 pre_gate: Vec::new(),
17131 phase,
17132 failure: None,
17133 }
17134 }
17135
17136 #[tokio::test]
17137 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
17138 crate::run::pin_test_home();
17139 let mut runner = runner_at(RunStatus::Landing);
17140 runner.state.merge_repairs.push(merge_repair_request(
17141 crate::run::MergeRepairPhase::Requested,
17142 ));
17143 runner.state.save().unwrap();
17144 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
17145 assert!(
17146 runner
17147 .execute()
17148 .await
17149 .unwrap_err()
17150 .to_string()
17151 .contains("already running")
17152 );
17153 assert_eq!(
17154 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
17155 crate::run::MergeRepairPhase::Requested
17156 );
17157 drop(claim);
17158 let mut disk = runner.state.clone();
17159 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
17160 disk.save().unwrap();
17161 assert!(
17162 runner
17163 .execute()
17164 .await
17165 .unwrap_err()
17166 .to_string()
17167 .contains("decision changed")
17168 );
17169 assert_eq!(
17170 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
17171 crate::run::MergeRepairPhase::Complete
17172 );
17173 }
17174
17175 #[test]
17176 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
17177 crate::run::pin_test_home();
17178 let mut runner = runner_at(RunStatus::Landing);
17179 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
17180 runner.state.reviews = request.reviews.clone();
17181 runner.state.gate = request.gate.clone();
17182 runner.state.gate_ran = true;
17183 runner.state.land_approval = Some(crate::run::LandApproval {
17184 question: "old".into(),
17185 head: "old-head".into(),
17186 });
17187 runner.state.merge_repairs.push(request);
17188 runner.begin_merge_repair_review(0).unwrap();
17189 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
17190 assert!(!runner.state.gate_status().ok());
17191 assert!(runner.state.land_approval.is_none());
17192 let disk = RunState::load(&runner.state.id).unwrap();
17193 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
17194 assert!(disk.merge_repairs[0].gate[0].ok());
17195 assert_eq!(
17196 disk.merge_repairs[0].phase,
17197 crate::run::MergeRepairPhase::Reviewing
17198 );
17199 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
17200 assert!(disk.reviews.is_empty());
17201 }
17202
17203 #[tokio::test]
17204 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
17205 crate::run::pin_test_home();
17206 for phase in [
17207 crate::run::MergeRepairPhase::Fixing,
17208 crate::run::MergeRepairPhase::Requested,
17209 ] {
17210 let mut runner = runner_at(RunStatus::Landing);
17211 runner.state.config.graph.land_rounds = 0;
17212 runner.state.merge_repairs.push(merge_repair_request(phase));
17213 runner.state.save().unwrap();
17214 assert!(!runner.repair_merge_review().await.unwrap());
17215 assert_eq!(runner.state.status, RunStatus::Blocked);
17216 assert_eq!(
17217 runner.state.merge_repairs[0].phase,
17218 crate::run::MergeRepairPhase::Failed
17219 );
17220 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
17221 assert!(
17222 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
17223 "interrupted"
17224 } else {
17225 "budget exhausted"
17226 })
17227 );
17228 assert!(runner.state.merge.is_none());
17229 }
17230 }
17231
17232 #[test]
17236 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
17237 crate::run::pin_test_home();
17238 let mut runner = runner_at(RunStatus::Implementing);
17239 let interrupt = Pause::new();
17240 runner.watch_interrupt(interrupt.clone());
17241
17242 interrupt.park_because("task a1b2 asked to run first");
17243
17244 assert!(runner.park_here().expect("park_here"));
17245 assert!(runner.state.parked);
17246 let last = runner.state.events.last().expect("a park event");
17247 assert_eq!(last.node, "park");
17248 assert!(
17249 last.message.contains("task a1b2 asked to run first"),
17250 "expected the interrupt reason in {:?}",
17251 last.message
17252 );
17253 }
17254
17255 #[test]
17258 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
17259 crate::run::pin_test_home();
17260 let mut runner = runner_at(RunStatus::Implementing);
17261 runner.state.begin_pass();
17262 let shutdown = Pause::new();
17263 runner.on_pause(shutdown.clone());
17264 let upgrade = ParkCause::Upgrade {
17265 from: "0.121.0".to_owned(),
17266 to: Some("0.121.1".to_owned()),
17267 };
17268 shutdown.park_for(upgrade.clone());
17269 shutdown.park_for(ParkCause::Operator);
17270
17271 assert!(runner.park_here().expect("park_here"));
17272 assert_eq!(
17273 runner.state.passes[0].end,
17274 Some(crate::run::PassEnd::Parked { cause: upgrade })
17275 );
17276 let last = runner.state.events.last().expect("a park event");
17277 assert!(last.message.contains("0.121.0"), "{}", last.message);
17278 }
17279
17280 #[tokio::test]
17281 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
17282 crate::run::pin_test_home();
17283 let mut runner = runner_at(RunStatus::Reviewing);
17284 runner.state.save().unwrap();
17285 let mut execution = runner.claim_driver().await.expect("first execution");
17286 assert_eq!(execution.runner.state.passes.len(), 1);
17287 execution
17288 .driver
17289 .finish(&mut execution.runner.state)
17290 .await
17291 .unwrap();
17292 }
17293
17294 #[tokio::test]
17295 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
17296 crate::run::pin_test_home();
17297 let mut runner = runner_at(RunStatus::Reviewing);
17298 runner.state.config.agents = vec![conductor()];
17299 runner.state.config.graph.timeout_resource_wait = 0;
17300 runner.state.save().unwrap();
17301 let id = runner.state.id.clone();
17302 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
17303 assert!(runner.claim_driver().await.is_err());
17304 drop(owner);
17305 drop(runner);
17306 let mut resumed = Runner::resume(&id).unwrap();
17307 assert_eq!(resumed.state.driver_claimed, Some(false));
17308 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
17309 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
17310 let mut execution = resumed
17311 .claim_driver()
17312 .await
17313 .expect("retry initial claim after reload");
17314 assert_eq!(execution.runner.state.driver_claimed, Some(true));
17315 assert_eq!(execution.runner.state.passes.len(), 1);
17316 execution
17317 .driver
17318 .finish(&mut execution.runner.state)
17319 .await
17320 .unwrap();
17321 }
17322
17323 #[tokio::test]
17324 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
17325 crate::run::pin_test_home();
17326 let mut runner = runner_at(RunStatus::Reviewing);
17327 runner.state.config.agents = vec![conductor()];
17328 runner.state.save().unwrap();
17329 let id = runner.state.id.clone();
17330 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
17331 let mut resumed = Runner::resume(&id).unwrap();
17332 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
17333 assert!(resumed.claim_driver().await.is_err());
17334 assert!(RunState::load(&id).unwrap().passes.is_empty());
17335 drop(lease);
17336 let mut execution = resumed.claim_driver().await.unwrap();
17337 execution
17338 .driver
17339 .finish(&mut execution.runner.state)
17340 .await
17341 .unwrap();
17342 }
17343
17344 #[tokio::test]
17345 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
17346 crate::run::pin_test_home();
17347 let mut runner = runner_at(RunStatus::Reviewing);
17348 let state = &mut runner.state;
17349 state.config.agents = vec![conductor()];
17350 state.driver_claimed = None;
17351 state.driver_pid = Some(std::process::id());
17352 state.driver_started_at = None;
17353 state.begin_pass();
17354 state.save().unwrap();
17355 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
17356 std::fs::write(
17357 state.dir().join("driver.json"),
17358 serde_json::to_vec(&serde_json::json!({
17359 "pid": state.driver_pid, "started_at": null,
17360 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
17361 }))
17362 .unwrap(),
17363 )
17364 .unwrap();
17365 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
17366 drop(lease);
17367 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
17368 let mut resumed = Runner::resume(&state.id).unwrap();
17369 assert!(!resumed.state.driver_exited);
17370 let mut execution = resumed
17371 .claim_driver()
17372 .await
17373 .expect("lease proves previous execution ended");
17374 assert_eq!(execution.runner.state.passes.len(), 2);
17375 execution
17376 .driver
17377 .finish(&mut execution.runner.state)
17378 .await
17379 .unwrap();
17380 }
17381
17382 #[tokio::test]
17383 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
17384 crate::run::pin_test_home();
17385 let mut runner = runner_at(RunStatus::Reviewing);
17386 let mut json = serde_json::to_value(&runner.state).unwrap();
17387 json.as_object_mut().unwrap().remove("driver_claimed");
17388 runner.state = serde_json::from_value(json).unwrap();
17389 assert_eq!(runner.state.driver_claimed, None);
17390 runner.state.save().unwrap();
17391 let mut execution = runner
17392 .claim_driver()
17393 .await
17394 .expect("the execution lease, not a missing marker, excludes a second driver");
17395 assert_eq!(execution.runner.state.passes.len(), 1);
17396 execution
17397 .driver
17398 .finish(&mut execution.runner.state)
17399 .await
17400 .unwrap();
17401 }
17402
17403 #[tokio::test]
17404 async fn driver_admission_refuses_a_proven_live_driver() {
17405 crate::run::pin_test_home();
17406 let mut runner = runner_at(RunStatus::Reviewing);
17407 let pid = std::process::id();
17408 runner.state.driver_pid = Some(pid);
17409 runner.state.driver_started_at = crate::proc::process_started_at(pid);
17410 assert!(runner.state.driver_started_at.is_some());
17411 runner.state.save().unwrap();
17412 let error = runner
17413 .claim_driver()
17414 .await
17415 .err()
17416 .expect("a live driver must refuse");
17417 assert!(error.to_string().contains("live driver"), "{error:#}");
17418 let saved = RunState::load(&runner.state.id).unwrap();
17419 assert_eq!(saved.driver_pid, Some(pid));
17420 assert_eq!(saved.status, RunStatus::Reviewing);
17421 assert!(saved.passes.is_empty());
17422 }
17423
17424 #[test]
17425 fn parking_resource_halt_persists_the_pass_once() {
17426 crate::run::pin_test_home();
17427 let mut runner = runner_at(RunStatus::Reviewing);
17428 runner.state.begin_pass();
17429 runner.state.resource_block = Some(crate::run::ResourceBlock {
17430 node: "fix".into(),
17431 kind: "shared build cache".into(),
17432 detail: "cancelled".into(),
17433 at: Timestamp::now(),
17434 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
17435 });
17436 let cause = ParkCause::Upgrade {
17437 from: "old".into(),
17438 to: Some("new".into()),
17439 };
17440 runner.pause.park_for(cause.clone());
17441 runner.stop_for_resource().unwrap();
17444 assert!(runner.resource_halt().unwrap());
17445 assert!(runner.park_here().unwrap());
17446 let saved = RunState::load(&runner.state.id).unwrap();
17447 assert!(saved.parked);
17448 assert_eq!(saved.status, RunStatus::Reviewing);
17449 assert!(saved.passes[0].ended_at.is_some());
17450 assert_eq!(
17451 saved.passes[0].end,
17452 Some(crate::run::PassEnd::Parked {
17453 cause: cause.clone()
17454 })
17455 );
17456 assert_eq!(
17457 saved.resource_block.unwrap().cause,
17458 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
17459 );
17460 assert_eq!(
17461 saved
17462 .events
17463 .iter()
17464 .filter(|event| event.node == "park")
17465 .count(),
17466 1
17467 );
17468 }
17469
17470 #[test]
17478 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
17479 crate::run::pin_test_home();
17480 let mut runner = runner_at(RunStatus::Implementing);
17481 let shutdown = Pause::new();
17482 runner.on_pause(shutdown.clone());
17483 let interrupt = Pause::new();
17484 runner.watch_interrupt(interrupt.clone());
17485
17486 assert!(!runner.park_here().expect("park_here"));
17488 assert!(!runner.state.parked);
17489
17490 interrupt.park_because("test");
17492 assert!(!shutdown.parked());
17493 assert!(runner.park_here().expect("park_here"));
17494 }
17495
17496 #[tokio::test]
17510 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
17511 crate::run::pin_test_home();
17512 let mut runner = runner_at(RunStatus::Implementing);
17513 let interrupt = Pause::new();
17514 runner.watch_interrupt(interrupt.clone());
17515
17516 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
17517 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
17518
17519 let node = async move {
17523 started_tx.send(()).expect("send started");
17524 finish_rx.await.expect("recv finish");
17525 "node finished"
17526 };
17527
17528 let interrupter = async move {
17529 started_rx.await.expect("recv started");
17530 interrupt.park_because("higher-priority task waiting");
17532 tokio::task::yield_now().await;
17536 finish_tx.send(()).expect("send finish");
17537 };
17538
17539 let (node_result, ()) = tokio::join!(node, interrupter);
17540 assert_eq!(
17541 node_result, "node finished",
17542 "the in-flight call ran to completion"
17543 );
17544
17545 assert!(runner.park_here().expect("park_here"));
17548 assert!(runner.state.parked);
17549 }
17550
17551 #[test]
17557 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
17558 crate::run::pin_test_home();
17559 let mut runner = runner_at(RunStatus::Judging);
17560 runner.state.config.agents = vec![conductor()];
17564 runner.state.candidates = vec![Candidate {
17565 index: 0,
17566 label: 'A',
17567 agent: "alpha".to_owned(),
17568 branch: "magi/x/A".to_owned(),
17569 worktree: PathBuf::from("/nonexistent/worktree"),
17570 summary: "did the thing".to_owned(),
17571 stat: "1 file changed".to_owned(),
17572 files: 1,
17573 commits: 1,
17574 empty: false,
17575 failed: None,
17576 verified_noop: None,
17577 duration_ms: 1234,
17578 folded: false,
17579 }];
17580 let run_id = runner.state.id.clone();
17581
17582 let interrupt = Pause::new();
17583 runner.watch_interrupt(interrupt.clone());
17584 interrupt.park_because("task c3d4 asked to run first");
17585 assert!(runner.park_here().expect("park_here"));
17586
17587 let resumed = Runner::resume(&run_id).expect("resume");
17588 assert_eq!(resumed.state.candidates.len(), 1);
17589 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
17590 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
17591 assert_eq!(resumed.state.status, runner.state.status);
17592 assert!(
17593 resumed.state.parked,
17594 "still parked until `execute` actually walks the graph again"
17595 );
17596 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
17597 }
17598
17599 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
17601 let mut q = ask::Question::new(
17602 run.to_owned(),
17603 "implement".to_owned(),
17604 "impl-A".to_owned(),
17605 "Which storage backend should the cache use?".to_owned(),
17606 String::new(),
17607 vec!["SQLite".to_owned(), "Redis".to_owned()],
17608 );
17609 store.put(&mut q).unwrap();
17610 q
17611 }
17612
17613 #[test]
17614 fn a_failed_runs_open_question_is_abandoned() {
17615 ask_test_home();
17616 let store = ask::Questions::open();
17617 let mut runner = runner_at(RunStatus::Failed);
17618 let run = runner.state.id.clone();
17619 let q = ask_open_question(&store, &run);
17620
17621 runner.settle_questions();
17622
17623 let back = store.get(&q.id).unwrap();
17624 assert!(
17625 !back.status.open(),
17626 "the seat that asked died with the run; nobody is left to read an answer"
17627 );
17628 assert!(
17629 back.detail.contains(&run) && back.detail.contains("failed"),
17630 "the reason names what the run became, not just that it is gone: {}",
17631 back.detail
17632 );
17633 }
17634
17635 #[test]
17636 fn a_merged_runs_open_question_is_abandoned_too() {
17637 ask_test_home();
17638 let store = ask::Questions::open();
17639 for status in [RunStatus::Merged, RunStatus::Ready] {
17642 let mut runner = runner_at(status);
17643 let run = runner.state.id.clone();
17644 let q = ask_open_question(&store, &run);
17645
17646 runner.settle_questions();
17647
17648 let back = store.get(&q.id).unwrap();
17649 assert!(
17650 !back.status.open(),
17651 "{status:?} run's question must not outlive the run"
17652 );
17653 }
17654 }
17655
17656 #[test]
17657 fn a_still_resumable_runs_open_question_is_left_alone() {
17658 ask_test_home();
17659 let store = ask::Questions::open();
17660 for status in [RunStatus::Blocked, RunStatus::Stalled] {
17666 let mut runner = runner_at(status);
17667 let run = runner.state.id.clone();
17668 let q = ask_open_question(&store, &run);
17669
17670 runner.settle_questions();
17671
17672 let back = store.get(&q.id).unwrap();
17673 assert!(
17674 back.status.open(),
17675 "{status:?} is still alive; the question must still be waiting"
17676 );
17677 }
17678 }
17679
17680 #[test]
17681 fn settle_questions_never_touches_an_already_answered_question() {
17682 ask_test_home();
17683 let store = ask::Questions::open();
17684 let mut runner = runner_at(RunStatus::Failed);
17685 let run = runner.state.id.clone();
17686 let mut q = ask_open_question(&store, &run);
17687 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
17688 .unwrap();
17689 store.put(&mut q).unwrap();
17690
17691 runner.settle_questions();
17696 runner.settle_questions();
17697
17698 let back = store.get(&q.id).unwrap();
17699 assert_eq!(
17700 back.status,
17701 ask::QuestionStatus::Answered,
17702 "a real answer is a decision on record, never overwritten by a sweep"
17703 );
17704 }
17705
17706 #[tokio::test]
17717 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
17718 crate::run::pin_test_home();
17719 let tmp = tempfile::tempdir().expect("tempdir");
17720 let repo = tmp.path().join("repo");
17721 std::fs::create_dir_all(&repo).unwrap();
17722 init_repo(&repo);
17723
17724 let mut config = Config::default();
17725 config.graph.worktree_root = Some(tmp.path().join("wt"));
17726
17727 let mut state = RunState::new(
17728 repo.clone(),
17729 "main".to_owned(),
17730 "deadbeef".to_owned(),
17731 "task".to_owned(),
17732 config,
17733 );
17734 let root = state.worktree_root();
17735 let wt_a = root.join("cand-A");
17736 let wt_b = root.join("cand-B");
17737 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
17738 .await
17739 .expect("worktree A");
17740 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
17741 .await
17742 .expect("worktree B");
17743
17744 state.candidates = vec![
17745 Candidate {
17746 index: 0,
17747 label: 'A',
17748 agent: "alpha".to_owned(),
17749 branch: "magi/x/A".to_owned(),
17750 worktree: wt_a.clone(),
17751 summary: String::new(),
17752 stat: String::new(),
17753 files: 0,
17754 commits: 0,
17755 empty: false,
17756 failed: None,
17757 verified_noop: None,
17758 duration_ms: 0,
17759 folded: false,
17760 },
17761 Candidate {
17762 index: 1,
17763 label: 'B',
17764 agent: "beta".to_owned(),
17765 branch: "magi/x/B".to_owned(),
17766 worktree: wt_b.clone(),
17767 summary: String::new(),
17768 stat: String::new(),
17769 files: 0,
17770 commits: 0,
17771 empty: false,
17772 failed: None,
17773 verified_noop: None,
17774 duration_ms: 0,
17775 folded: false,
17776 },
17777 ];
17778 state.tally = Some(Tally {
17779 first_choice: BTreeMap::from([('A', 1)]),
17780 borda: BTreeMap::new(),
17781 winner: 'A',
17782 rankings: 1,
17783 unanimous_initial: true,
17784 deliberated: false,
17785 changed_votes: 0,
17786 unanimous_final: true,
17787 tie_break: None,
17788 judges: 1,
17789 present: 1,
17790 quorum: 1,
17791 met_quorum: true,
17792 uncontested: None,
17793 });
17794 state.status = RunStatus::Ready;
17795
17796 fold_run(&mut state, false, &crate::run::home())
17797 .await
17798 .expect("fold_run");
17799
17800 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17801 assert!(
17802 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17803 "the unmerged winner's branch survives"
17804 );
17805 assert!(
17806 !state.candidates[0].folded,
17807 "the winner is not marked folded"
17808 );
17809
17810 assert!(!wt_b.exists(), "the loser's worktree is removed");
17811 assert!(
17812 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17813 "the loser's branch is removed"
17814 );
17815 assert!(state.candidates[1].folded, "the loser is marked folded");
17816 }
17817
17818 #[tokio::test]
17819 #[cfg(unix)]
17820 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17821 use std::os::fd::AsRawFd as _;
17822 use std::time::Duration;
17823
17824 let tmp = tempfile::tempdir().unwrap();
17825 let repo = tmp.path().join("repo");
17826 std::fs::create_dir_all(&repo).unwrap();
17827 init_repo(&repo);
17828 let common = repo.join(".git");
17829 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17830 let home = tmp.path().join("home");
17831 let mut config = Config::default();
17832 config.coordination = crate::config::test_coordination();
17833 config.graph.worktree_root = Some(tmp.path().join("wt"));
17834 config.graph.timeout_resource_wait = 1;
17835 let mut state = RunState::new(
17836 repo.clone(),
17837 "main".into(),
17838 "deadbeef".into(),
17839 "task".into(),
17840 config,
17841 );
17842 let root = state.worktree_root();
17843 let candidate = root.join("cand-A");
17844 let reviewer = root.join("review-1");
17845 let branch = state.branch_for('A');
17846 git::worktree_add_branch(&repo, &candidate, &branch, "main")
17847 .await
17848 .unwrap();
17849 git::worktree_add_detached(&repo, &reviewer, "main")
17850 .await
17851 .unwrap();
17852 git::acquire_worktree_config(&repo).await.unwrap();
17853 state.enabled_worktree_config = true;
17854 state.status = RunStatus::Ready;
17855 state.candidates.push(Candidate {
17856 index: 0,
17857 label: 'A',
17858 agent: "implement".into(),
17859 branch: branch.clone(),
17860 worktree: candidate.clone(),
17861 summary: String::new(),
17862 stat: String::new(),
17863 files: 0,
17864 commits: 0,
17865 empty: false,
17866 failed: None,
17867 verified_noop: None,
17868 duration_ms: 0,
17869 folded: false,
17870 });
17871 state.save_under(&home).unwrap();
17872 let config_before = std::fs::read(common.join("config")).unwrap();
17873 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17874 .await
17875 .unwrap();
17876 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17879 assert_eq!(
17880 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17881 0
17882 );
17883 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17884 assert!(
17885 tokio::time::timeout(Duration::from_millis(200), &mut fold)
17886 .await
17887 .is_err()
17888 );
17889 assert!(candidate.exists());
17890 assert!(reviewer.exists());
17891 assert!(git::branch_exists(&repo, &branch).await.unwrap());
17892 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17893 assert_eq!(
17894 git::git(&repo, &["worktree", "list", "--porcelain"])
17895 .await
17896 .unwrap(),
17897 worktrees_before
17898 );
17899 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17900 .await
17901 .unwrap()
17902 .unwrap_err();
17903 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17904 drop(fold);
17905 assert!(candidate.exists() && reviewer.exists());
17906 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17907 assert_eq!(
17908 state.resource_block.as_ref().unwrap().kind,
17909 "repository coordination"
17910 );
17911 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17912 drop(external);
17913 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17914 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17915 .await
17916 .unwrap()
17917 .unwrap();
17918 drop(fold);
17919 assert!(removed.contains(&branch));
17920 assert!(!root.exists());
17921 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17922 assert!(state.candidates[0].folded);
17923 assert!(!state.enabled_worktree_config);
17924 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17925 let enabled = git::git_raw(
17926 &repo,
17927 &["config", "--local", "--get", "extensions.worktreeConfig"],
17928 )
17929 .await
17930 .unwrap();
17931 assert!(!enabled.ok(), "the last config reference was released");
17932 }
17933
17934 #[tokio::test]
17937 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17938 let tmp = tempfile::tempdir().expect("tempdir");
17939 let repo = tmp.path().join("repo");
17940 std::fs::create_dir_all(&repo).unwrap();
17941 init_repo(&repo);
17942 let home = tmp.path().join("home");
17943
17944 let mut config = Config::default();
17945 config.graph.worktree_root = Some(tmp.path().join("wt"));
17946 let mut state = RunState::new(
17947 repo.clone(),
17948 "main".to_owned(),
17949 "deadbeef".to_owned(),
17950 "task".to_owned(),
17951 config,
17952 );
17953 git::git(&repo, &["branch", "magi/x/A", "main"])
17955 .await
17956 .expect("branch");
17957 state.candidates = vec![Candidate {
17958 index: 0,
17959 label: 'A',
17960 agent: "alpha".to_owned(),
17961 branch: "magi/x/A".to_owned(),
17962 worktree: state.worktree_root().join("cand-A"),
17963 summary: String::new(),
17964 stat: String::new(),
17965 files: 0,
17966 commits: 0,
17967 empty: false,
17968 failed: None,
17969 verified_noop: None,
17970 duration_ms: 0,
17971 folded: true,
17972 }];
17973 state.released_to = Some("20260901-000000-new1".to_owned());
17974 state.released_branches = vec!["magi/x/A".to_owned()];
17975
17976 fold_run(&mut state, true, &home).await.expect("fold_run");
17977 state.status = RunStatus::Merged;
17978 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17979 state.merge = Some(crate::run::MergeOutcome {
17980 mode: MergeMode::Pr,
17981 ok: true,
17982 detail: "confirmed merged".into(),
17983 empty: false,
17984 });
17985 fold_run(&mut state, true, &home)
17986 .await
17987 .expect("merged fold");
17988
17989 assert!(
17990 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17991 "the handed-over branch survives a fold"
17992 );
17993 }
17994
17995 #[tokio::test]
17999 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
18000 let tmp = tempfile::tempdir().expect("tempdir");
18001 let repo = tmp.path().join("repo");
18002 std::fs::create_dir_all(&repo).unwrap();
18003 init_repo(&repo);
18004 let run = |args: &[&str]| {
18005 let out = std::process::Command::new("git")
18006 .quiet()
18007 .args(args)
18008 .current_dir(&repo)
18009 .output()
18010 .expect("spawn git");
18011 assert!(out.status.success(), "git {args:?}");
18012 };
18013 run(&["branch", "magi/x/A"]);
18014 run(&["checkout", "-q", "-b", "magi/x/B"]);
18015 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
18016 run(&["add", "-A"]);
18017 run(&["commit", "-q", "-m", "work"]);
18018 run(&["checkout", "-q", "main"]);
18019
18020 let probe = RunState::new(
18024 repo.clone(),
18025 "main".to_owned(),
18026 "deadbeef".to_owned(),
18027 "task".to_owned(),
18028 Config::default(),
18029 );
18030 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
18031 let bare = tmp.path().join("origin.git");
18032 let out = std::process::Command::new("git")
18033 .quiet()
18034 .args(["init", "-q", "--bare"])
18035 .arg(&bare)
18036 .output()
18037 .expect("spawn git");
18038 assert!(out.status.success(), "git init --bare");
18039 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
18040 run(&["push", "-q", "origin", "main"]);
18041
18042 let mut state = RunState::new(
18043 repo.clone(),
18044 "main".to_owned(),
18045 "deadbeef".to_owned(),
18046 "task".to_owned(),
18047 Config::default(),
18048 );
18049 state.seeds = vec![refs::Seed {
18050 token: "magi/27b2/A".to_owned(),
18051 kind: refs::SeedKind::Unresolved,
18052 sha: String::new(),
18053 branch: true,
18054 detail: "no branch or commit named magi/27b2/A".to_owned(),
18055 }];
18056
18057 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
18058 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
18059 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
18060 let detail = empty_candidate_detail(&state, "main");
18061 assert!(detail.starts_with("empty candidate"), "{detail}");
18062 assert!(!detail.contains("magi/27b2/A"), "{detail}");
18063 assert!(detail.contains("--start-from"), "{detail}");
18064 }
18065
18066 #[tokio::test]
18075 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
18076 let tmp = tempfile::tempdir().expect("tempdir");
18077 let repo = tmp.path().join("repo");
18078 std::fs::create_dir_all(&repo).unwrap();
18079 init_repo(&repo);
18080
18081 let mut config = Config::default();
18082 config.merge.mode = MergeMode::Local;
18083
18084 let mut state = RunState::new(
18085 repo.clone(),
18086 "main".to_owned(),
18087 "deadbeef".to_owned(),
18088 "task".to_owned(),
18089 config,
18090 );
18091 state.candidates = vec![Candidate {
18092 index: 0,
18093 label: 'A',
18094 agent: "alpha".to_owned(),
18095 branch: "does-not-exist".to_owned(),
18096 worktree: repo.clone(),
18097 summary: String::new(),
18098 stat: String::new(),
18099 files: 0,
18100 commits: 0,
18101 empty: false,
18102 failed: None,
18103 verified_noop: None,
18104 duration_ms: 0,
18105 folded: false,
18106 }];
18107 state.tally = Some(Tally {
18108 first_choice: BTreeMap::from([('A', 1)]),
18109 borda: BTreeMap::new(),
18110 winner: 'A',
18111 rankings: 1,
18112 unanimous_initial: true,
18113 deliberated: false,
18114 changed_votes: 0,
18115 unanimous_final: true,
18116 tie_break: None,
18117 judges: 0,
18118 present: 0,
18119 quorum: 0,
18120 met_quorum: true,
18121 uncontested: Some("only candidate A produced a change".to_owned()),
18122 });
18123 state.reviews = vec![ReviewRound {
18124 round: 1,
18125 head: "deadbeef".to_owned(),
18126 verified_head: None,
18127 verified_at: None,
18128 reviews: Vec::new(),
18129 e2e: Vec::new(),
18130 fix: None,
18131 blocking: 0,
18132 answered: 0,
18133 expected: 0,
18134 clean: true,
18135 verify_retried: false,
18136 e2e_deferred: false,
18137 e2e_defer_reason: None,
18138 progressed: false,
18139 vote_split: false,
18140 reconsideration: Vec::new(),
18141 verdict: None,
18142 }];
18143 state.gate = vec![CommandOutcome {
18144 command: "test".to_owned(),
18145 code: Some(0),
18146 output_tail: String::new(),
18147 duration_ms: 0,
18148 resource_blocked: false,
18149 }];
18150 state.gate_ran = true;
18151 state.status = RunStatus::Ready;
18156 state.merge = Some(MergeOutcome {
18157 mode: MergeMode::Local,
18158 ok: false,
18159 detail: "already concluded".to_owned(),
18160 empty: false,
18161 });
18162
18163 let mut runner = Runner {
18164 state,
18165 roles: ResolvedRoles {
18166 implementers: Vec::new(),
18167 judges: Vec::new(),
18168 reviewers: Vec::new(),
18169 fixer: None,
18170 conductor: conductor(),
18171 implementer_roster: Vec::new(),
18172 judge_roster: Vec::new(),
18173 reviewer_roster: Vec::new(),
18174 },
18175 sem: Arc::new(Semaphore::new(1)),
18176 pause: Pause::new(),
18177 interrupt: Pause::new(),
18178 };
18179
18180 runner.merge().await.expect("merge");
18181
18182 assert_eq!(
18183 runner.state.status,
18184 RunStatus::Ready,
18185 "a concluded run's status must not change on reentry"
18186 );
18187 assert_eq!(
18188 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18189 Some("already concluded"),
18190 "merge must not run again once the node already recorded an outcome"
18191 );
18192 }
18193
18194 #[tokio::test]
18203 async fn merge_refuses_a_gate_that_has_not_actually_run() {
18204 let tmp = tempfile::tempdir().expect("tempdir");
18205 let repo = tmp.path().join("repo");
18206 std::fs::create_dir_all(&repo).unwrap();
18207 init_repo(&repo);
18208
18209 let mut config = Config::default();
18210 config.merge.mode = MergeMode::Local;
18211
18212 let mut state = RunState::new(
18213 repo.clone(),
18214 "main".to_owned(),
18215 "deadbeef".to_owned(),
18216 "task".to_owned(),
18217 config,
18218 );
18219 state.candidates = vec![Candidate {
18220 index: 0,
18221 label: 'A',
18222 agent: "alpha".to_owned(),
18223 branch: "does-not-exist".to_owned(),
18224 worktree: repo.clone(),
18225 summary: String::new(),
18226 stat: String::new(),
18227 files: 0,
18228 commits: 0,
18229 empty: false,
18230 failed: None,
18231 verified_noop: None,
18232 duration_ms: 0,
18233 folded: false,
18234 }];
18235 state.tally = Some(Tally {
18236 first_choice: BTreeMap::from([('A', 1)]),
18237 borda: BTreeMap::new(),
18238 winner: 'A',
18239 rankings: 1,
18240 unanimous_initial: true,
18241 deliberated: false,
18242 changed_votes: 0,
18243 unanimous_final: true,
18244 tie_break: None,
18245 judges: 0,
18246 present: 0,
18247 quorum: 0,
18248 met_quorum: true,
18249 uncontested: Some("only candidate A produced a change".to_owned()),
18250 });
18251 state.reviews = vec![ReviewRound {
18252 round: 1,
18253 head: "deadbeef".to_owned(),
18254 verified_head: None,
18255 verified_at: None,
18256 reviews: Vec::new(),
18257 e2e: Vec::new(),
18258 fix: None,
18259 blocking: 0,
18260 answered: 0,
18261 expected: 0,
18262 clean: true,
18263 verify_retried: false,
18264 e2e_deferred: false,
18265 e2e_defer_reason: None,
18266 progressed: false,
18267 vote_split: false,
18268 reconsideration: Vec::new(),
18269 verdict: None,
18270 }];
18271 state.gate = Vec::new();
18273 state.gate_ran = false;
18274 state.status = RunStatus::Gating;
18275
18276 let mut runner = Runner {
18277 state,
18278 roles: ResolvedRoles {
18279 implementers: Vec::new(),
18280 judges: Vec::new(),
18281 reviewers: Vec::new(),
18282 fixer: None,
18283 conductor: conductor(),
18284 implementer_roster: Vec::new(),
18285 judge_roster: Vec::new(),
18286 reviewer_roster: Vec::new(),
18287 },
18288 sem: Arc::new(Semaphore::new(1)),
18289 pause: Pause::new(),
18290 interrupt: Pause::new(),
18291 };
18292
18293 runner.merge().await.expect("merge");
18294
18295 assert!(
18296 runner.state.merge.is_none(),
18297 "an empty gate must never be read as a passing one: {:?}",
18298 runner.state.merge
18299 );
18300 }
18301
18302 #[tokio::test]
18309 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
18310 let tmp = tempfile::tempdir().expect("tempdir");
18311 let repo = tmp.path().join("repo");
18312 std::fs::create_dir_all(&repo).unwrap();
18313 init_repo(&repo);
18314
18315 let config = Config::default();
18317
18318 let mut state = RunState::new(
18319 repo.clone(),
18320 "main".to_owned(),
18321 "deadbeef".to_owned(),
18322 "task".to_owned(),
18323 config,
18324 );
18325 state.candidates = vec![Candidate {
18326 index: 0,
18327 label: 'A',
18328 agent: "alpha".to_owned(),
18329 branch: "does-not-exist".to_owned(),
18330 worktree: repo.clone(),
18331 summary: String::new(),
18332 stat: String::new(),
18333 files: 0,
18334 commits: 0,
18335 empty: false,
18336 failed: None,
18337 verified_noop: None,
18338 duration_ms: 0,
18339 folded: false,
18340 }];
18341 state.tally = Some(Tally {
18342 first_choice: BTreeMap::from([('A', 1)]),
18343 borda: BTreeMap::new(),
18344 winner: 'A',
18345 rankings: 1,
18346 unanimous_initial: true,
18347 deliberated: false,
18348 changed_votes: 0,
18349 unanimous_final: true,
18350 tie_break: None,
18351 judges: 0,
18352 present: 0,
18353 quorum: 0,
18354 met_quorum: true,
18355 uncontested: Some("only candidate A produced a change".to_owned()),
18356 });
18357 state.reviews = vec![ReviewRound {
18358 round: 1,
18359 head: "deadbeef".to_owned(),
18360 verified_head: None,
18361 verified_at: None,
18362 reviews: Vec::new(),
18363 e2e: Vec::new(),
18364 fix: None,
18365 blocking: 0,
18366 answered: 0,
18367 expected: 0,
18368 clean: true,
18369 verify_retried: false,
18370 e2e_deferred: false,
18371 e2e_defer_reason: None,
18372 progressed: false,
18373 vote_split: false,
18374 reconsideration: Vec::new(),
18375 verdict: None,
18376 }];
18377
18378 let mut runner = Runner {
18379 state,
18380 roles: ResolvedRoles {
18381 implementers: Vec::new(),
18382 judges: Vec::new(),
18383 reviewers: Vec::new(),
18384 fixer: None,
18385 conductor: conductor(),
18386 implementer_roster: Vec::new(),
18387 judge_roster: Vec::new(),
18388 reviewer_roster: Vec::new(),
18389 },
18390 sem: Arc::new(Semaphore::new(1)),
18391 pause: Pause::new(),
18392 interrupt: Pause::new(),
18393 };
18394
18395 runner.gate().await.expect("gate");
18396 assert!(
18397 runner.state.gate_ran,
18398 "zero configured commands is still a real attempt, not an unrun gate"
18399 );
18400 assert!(runner.state.gate.is_empty());
18401 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
18402 assert_ne!(
18403 runner.state.status,
18404 RunStatus::Blocked,
18405 "a gate with nothing to check must not read as failed"
18406 );
18407
18408 runner.merge().await.expect("merge");
18409 assert_eq!(
18410 runner.state.status,
18411 RunStatus::Ready,
18412 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
18413 );
18414 }
18415
18416 #[tokio::test]
18427 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
18428 crate::run::pin_test_home();
18429 let home = crate::run::home();
18430
18431 let tmp = tempfile::tempdir().expect("tempdir");
18432 let repo = tmp.path().join("repo");
18433 std::fs::create_dir_all(&repo).unwrap();
18434 init_repo(&repo);
18435 let cache_dir = tmp.path().join("target");
18438
18439 let mut config = Config::default();
18440 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
18441 config.graph.timeout_verify = Some(2);
18445 config.graph.timeout_resource_wait = 0;
18446
18447 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18448 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18449 .expect("no io error acquiring directly")
18450 {
18451 crate::cache::AcquireOutcome::Acquired(g) => g,
18452 crate::cache::AcquireOutcome::Busy(b) => {
18453 panic!("expected the direct acquire to win the lease first: {b:?}")
18454 }
18455 };
18456
18457 let mut state = RunState::new(
18458 repo.clone(),
18459 "main".to_owned(),
18460 "deadbeef".to_owned(),
18461 "task".to_owned(),
18462 config,
18463 );
18464 state.candidates = vec![Candidate {
18465 index: 0,
18466 label: 'A',
18467 agent: "alpha".to_owned(),
18468 branch: "does-not-exist".to_owned(),
18469 worktree: repo.clone(),
18470 summary: String::new(),
18471 stat: String::new(),
18472 files: 0,
18473 commits: 0,
18474 empty: false,
18475 failed: None,
18476 verified_noop: None,
18477 duration_ms: 0,
18478 folded: false,
18479 }];
18480 state.tally = Some(Tally {
18481 first_choice: BTreeMap::from([('A', 1)]),
18482 borda: BTreeMap::new(),
18483 winner: 'A',
18484 rankings: 1,
18485 unanimous_initial: true,
18486 deliberated: false,
18487 changed_votes: 0,
18488 unanimous_final: true,
18489 tie_break: None,
18490 judges: 0,
18491 present: 0,
18492 quorum: 0,
18493 met_quorum: true,
18494 uncontested: Some("only candidate A produced a change".to_owned()),
18495 });
18496 state.reviews = vec![ReviewRound {
18497 round: 1,
18498 head: "deadbeef".to_owned(),
18499 verified_head: None,
18500 verified_at: None,
18501 reviews: Vec::new(),
18502 e2e: Vec::new(),
18503 fix: None,
18504 blocking: 0,
18505 answered: 0,
18506 expected: 0,
18507 clean: true,
18508 verify_retried: false,
18509 e2e_deferred: false,
18510 e2e_defer_reason: None,
18511 progressed: false,
18512 vote_split: false,
18513 reconsideration: Vec::new(),
18514 verdict: None,
18515 }];
18516
18517 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
18518 state.reviews[0].e2e = vec![CommandOutcome {
18519 command: state.config.verify.e2e[0].clone(),
18520 code: Some(0),
18521 output_tail: String::new(),
18522 duration_ms: 0,
18523 resource_blocked: false,
18524 }];
18525 let mut runner = Runner {
18526 state,
18527 roles: ResolvedRoles {
18528 implementers: Vec::new(),
18529 judges: Vec::new(),
18530 reviewers: Vec::new(),
18531 fixer: None,
18532 conductor: conductor(),
18533 implementer_roster: Vec::new(),
18534 judge_roster: Vec::new(),
18535 reviewer_roster: Vec::new(),
18536 },
18537 sem: Arc::new(Semaphore::new(1)),
18538 pause: Pause::new(),
18539 interrupt: Pause::new(),
18540 };
18541
18542 runner.gate().await.expect("gate");
18543 assert!(runner.state.resource_block.is_none());
18544 assert!(
18545 !runner
18546 .state
18547 .events
18548 .iter()
18549 .any(|event| { event.message.contains("waiting for the shared build cache") }),
18550 "a gate with nothing to run must never request the held cache lease"
18551 );
18552 assert!(
18553 runner.state.gate_ran,
18554 "zero commands is still a real, immediate attempt"
18555 );
18556 assert!(runner.state.gate.is_empty());
18557 assert_ne!(
18558 runner.state.status,
18559 RunStatus::Blocked,
18560 "must not read as resource-blocked on a lease it never asked for"
18561 );
18562 }
18563
18564 #[cfg(unix)]
18567 #[tokio::test]
18568 async fn verification_is_standalone_inside_a_live_worker() {
18569 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
18570 if std::env::var_os(CHILD).is_none() {
18571 let home = tempfile::tempdir().unwrap();
18572 let generation = crate::worker::pin_current(home.path()).unwrap();
18573 let output = std::process::Command::new(std::env::current_exe().unwrap())
18574 .args([
18575 "--exact",
18576 "graph::tests::verification_is_standalone_inside_a_live_worker",
18577 "--nocapture",
18578 ])
18579 .env(CHILD, "1")
18580 .env("MAGI_HOME", home.path())
18581 .env("MAGI_WORKER_HOME", home.path())
18582 .env("MAGI_WORKER_ID", "abcdef")
18583 .env("MAGI_WORKER_GENERATION", &generation.id)
18584 .env("MAGI_WORKER_EXE", &generation.executable)
18585 .env("MAGI_RUN", "verification-parent")
18586 .env("MAGI_NODE", "gate")
18587 .env("MAGI_SEAT", "worker-fixture")
18588 .env("MAGI_PROCESS_OWNER", "fixture-owner")
18589 .env("MAGI_NO_AUTOUPDATE", "0")
18590 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
18591 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
18592 .env(
18593 "VERIFY_EXPECT_TOOLCHAIN",
18594 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
18595 )
18596 .env(
18597 "VERIFY_EXPECT_CACHE",
18598 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
18599 )
18600 .output()
18601 .unwrap();
18602 assert!(
18603 output.status.success()
18604 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
18605 "{}\n{}",
18606 String::from_utf8_lossy(&output.stdout),
18607 String::from_utf8_lossy(&output.stderr)
18608 );
18609 return;
18610 }
18611 let home = crate::run::home();
18612 let directory = home.join("workers/jobs/abcdef");
18613 std::fs::create_dir_all(&directory).unwrap();
18614 let job = crate::worker::Job {
18615 id: "abcdef".into(),
18616 sequence: 1,
18617 kind: "task".into(),
18618 key: "task:verification".into(),
18619 payload: serde_json::json!({}),
18620 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
18621 version: Some(env!("CARGO_PKG_VERSION").into()),
18622 status: crate::worker::Status::Running,
18623 result: None,
18624 pid: Some(std::process::id()),
18625 created_at: 0,
18626 finished_at: None,
18627 };
18628 std::fs::write(
18629 directory.join("job.json"),
18630 serde_json::to_vec(&job).unwrap(),
18631 )
18632 .unwrap();
18633 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
18634 let assert_worker = || {
18635 assert_eq!(
18636 crate::worker::current_job(&home).unwrap().unwrap().id,
18637 job.id
18638 );
18639 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
18640 assert_eq!(
18641 crate::worker::helper_executable(&home).unwrap(),
18642 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
18643 );
18644 assert!(
18645 crate::updater::Checker::new(&crate::config::Update {
18646 mode: crate::config::UpdateMode::Notify,
18647 interval: None,
18648 auto_restart: false,
18649 })
18650 .is_none(),
18651 "the driver remains pinned, even while tests are standalone"
18652 );
18653 };
18654 assert_worker();
18655 let repo = home.join("repo");
18656 std::fs::create_dir_all(&repo).unwrap();
18657 init_repo(&repo);
18658 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
18661 test -z "${MAGI_WORKER_GENERATION+x}" &&
18662 test -z "${MAGI_WORKER_HOME+x}" &&
18663 test -z "${MAGI_WORKER_EXE+x}" &&
18664 test "$MAGI_HOME/repo" -ef . &&
18665 test "$MAGI_RUN" = verification-parent &&
18666 test "$MAGI_NODE" = gate &&
18667 test "$MAGI_SEAT" = worker-fixture &&
18668 test -n "$MAGI_PROCESS_OWNER" &&
18669 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
18670 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
18671 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
18672 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
18673 assert!(
18675 !std::process::Command::new("sh")
18676 .args(["-c", probe])
18677 .current_dir(&repo)
18678 .status()
18679 .unwrap()
18680 .success()
18681 );
18682 let unrelated_home = tempfile::tempdir().unwrap();
18683 assert!(
18684 crate::worker::current_job(unrelated_home.path()).is_err(),
18685 "an inherited worker id cannot be resolved in a unit fixture's home"
18686 );
18687 let mut runner = runner_at(RunStatus::Gating);
18688 runner.state.repo = repo.clone();
18689 runner.state.config.disk.min_free_bytes = 0;
18690 runner.state.config.coordination = crate::config::test_coordination();
18691 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
18692 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
18693 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
18694 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
18695 let winner = Candidate {
18696 index: 0,
18697 label: 'A',
18698 agent: "fixture".into(),
18699 branch: "main".into(),
18700 worktree: repo.clone(),
18701 summary: String::new(),
18702 stat: String::new(),
18703 files: 0,
18704 commits: 0,
18705 empty: false,
18706 failed: None,
18707 verified_noop: None,
18708 duration_ms: 0,
18709 folded: false,
18710 };
18711 let (e2e, retried, pids) = run_e2e_with_retry(
18712 &mut runner.state,
18713 &["sh".into(), "-c".into()],
18714 &[format!("{probe} && printf initial")],
18715 &repo,
18716 Duration::from_secs(10),
18717 "initial",
18718 )
18719 .await;
18720 assert!(!retried && pids.is_empty());
18721 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
18722 assert_eq!(e2e[0].output_tail, "initial");
18723 let gate = runner.run_gate(&winner).await.unwrap();
18724 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
18725 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
18728 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
18729 passed: false,
18730 outcomes: Vec::new(),
18731 });
18732 runner.state.save().unwrap();
18733 let mut resumed = RunState::load(&runner.state.id).unwrap();
18734 let wait = crate::repo_lock::WaitPolicy::default();
18735 assert!(matches!(
18736 check_gate_repair_e2e(&mut resumed, &winner, &wait)
18737 .await
18738 .unwrap(),
18739 GateFix::Retry
18740 ));
18741 let saved = RunState::load(&resumed.id).unwrap();
18742 let repair = saved.gate_repair_e2e.unwrap();
18743 assert!(repair.passed);
18744 assert_eq!(repair.outcomes.len(), 1);
18745 assert!(repair.outcomes[0].ok());
18746 let landing = run_landing_gate(&mut resumed, &winner, &wait)
18747 .await
18748 .unwrap();
18749 assert_eq!(landing.len(), 1);
18750 assert!(landing[0].ok(), "{landing:?}");
18751 assert_eq!(
18752 std::fs::read_to_string(repo.join("phases")).unwrap(),
18753 "pre\ngate\ne2e\npre\ngate\n"
18754 );
18755 assert_worker();
18756 }
18757
18758 #[cfg(unix)]
18759 #[tokio::test]
18760 async fn verification_keeps_the_panic_and_the_full_output() {
18761 crate::run::pin_test_home();
18762 let tmp = tempfile::tempdir().unwrap();
18763 let mut config = Config::default();
18764 config.disk.min_free_bytes = 0;
18765 let mut state = RunState::new(
18766 tmp.path().to_owned(),
18767 "main".into(),
18768 "base".into(),
18769 "verification evidence".into(),
18770 config,
18771 );
18772 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18773 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
18774 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18775 let mut names = Vec::new();
18776 for _ in 0..2 {
18777 let (outcomes, _) = run_commands(
18778 &mut state,
18779 "gate",
18780 "gate",
18781 0,
18782 &["sh".into(), "-c".into()],
18783 &[script.into()],
18784 tmp.path(),
18785 Duration::from_secs(60),
18786 )
18787 .await;
18788 let tail = &outcomes[0].output_tail;
18789 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18790 assert!(tail.contains("boom assertion"));
18791 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18792 let line = tail.lines().next().unwrap();
18793 names.push(line.to_owned());
18794 }
18795 assert_ne!(names[0], names[1], "the second run must not overwrite");
18796 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18797 assert!(full.len() > 500_000 && full.contains("boom assertion"));
18798 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18799
18800 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18802 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18803 let (outcomes, _) = run_commands(
18804 &mut state,
18805 "gate",
18806 "gate",
18807 0,
18808 &["sh".into(), "-c".into()],
18809 &["echo failure-output; exit 7".into()],
18810 tmp.path(),
18811 Duration::from_secs(10),
18812 )
18813 .await;
18814 assert_eq!(outcomes[0].code, Some(7));
18815 assert!(state.events.iter().any(|e| {
18816 e.node == "gate" && e.message.contains("could not save the full gate output:")
18817 }));
18818 }
18819
18820 #[cfg(unix)]
18821 #[tokio::test]
18822 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18823 crate::run::pin_test_home();
18824 let tmp = tempfile::tempdir().unwrap();
18825 let mut config = Config::default();
18826 config.disk.min_free_bytes = 0;
18827 let mut state = RunState::new(
18828 tmp.path().to_owned(),
18829 "main".into(),
18830 "base".into(),
18831 "verification evidence".into(),
18832 config,
18833 );
18834 let (outcomes, pids) = run_commands(
18835 &mut state,
18836 "gate",
18837 "gate",
18838 1,
18839 &["sh".into(), "-c".into()],
18840 &[
18841 "printf before-timeout; exec sleep 30".into(),
18842 "kill -TERM $$".into(),
18843 "exit 7".into(),
18844 "printf success".into(),
18845 ],
18846 tmp.path(),
18847 Duration::from_secs(1),
18848 )
18849 .await;
18850 wait_for_timed_out_children_to_die(&pids).await;
18851 assert_eq!(
18852 pids.len(),
18853 1,
18854 "unleased timeout retains its death-check PID"
18855 );
18856 assert_eq!(outcomes.len(), 4);
18857 assert_eq!(outcomes[0].code, None);
18858 assert!(outcomes[0].output_tail.contains("before-timeout"));
18859 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18860 assert_eq!(outcomes[1].code, None);
18861 assert!(outcomes[1].output_tail.contains("command terminated:"));
18862 assert!(!outcomes[1].output_tail.contains("timed out"));
18863 assert_eq!(outcomes[2].code, Some(7));
18864 assert!(!outcomes[2].output_tail.contains("timed out"));
18865 assert_eq!(outcomes[3].code, Some(0));
18866 assert_eq!(outcomes[3].output_tail, "success");
18867 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18868 }
18869
18870 #[tokio::test]
18880 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18881 crate::run::pin_test_home();
18882
18883 let tmp = tempfile::tempdir().expect("tempdir");
18884 let repo = tmp.path().join("repo");
18885 std::fs::create_dir_all(&repo).unwrap();
18886 init_repo(&repo);
18887
18888 let mut config = Config::default();
18889 config.verify.gate = vec![
18890 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18891 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18892 .to_owned(),
18893 ];
18894
18895 let mut state = RunState::new(
18896 repo.clone(),
18897 "main".to_owned(),
18898 "deadbeef".to_owned(),
18899 "task".to_owned(),
18900 config,
18901 );
18902 let run_id = state.id.clone();
18903 state.candidates = vec![Candidate {
18904 index: 0,
18905 label: 'A',
18906 agent: "alpha".to_owned(),
18907 branch: "does-not-exist".to_owned(),
18908 worktree: repo.clone(),
18909 summary: String::new(),
18910 stat: String::new(),
18911 files: 0,
18912 commits: 0,
18913 empty: false,
18914 failed: None,
18915 verified_noop: None,
18916 duration_ms: 0,
18917 folded: false,
18918 }];
18919 state.tally = Some(Tally {
18920 first_choice: BTreeMap::from([('A', 1)]),
18921 borda: BTreeMap::new(),
18922 winner: 'A',
18923 rankings: 1,
18924 unanimous_initial: true,
18925 deliberated: false,
18926 changed_votes: 0,
18927 unanimous_final: true,
18928 tie_break: None,
18929 judges: 0,
18930 present: 0,
18931 quorum: 0,
18932 met_quorum: true,
18933 uncontested: Some("only candidate A produced a change".to_owned()),
18934 });
18935 state.reviews = vec![ReviewRound {
18936 round: 1,
18937 head: "deadbeef".to_owned(),
18938 verified_head: None,
18939 verified_at: None,
18940 reviews: Vec::new(),
18941 e2e: Vec::new(),
18942 fix: None,
18943 blocking: 0,
18944 answered: 0,
18945 expected: 0,
18946 clean: true,
18947 verify_retried: false,
18948 e2e_deferred: false,
18949 e2e_defer_reason: None,
18950 progressed: false,
18951 vote_split: false,
18952 reconsideration: Vec::new(),
18953 verdict: None,
18954 }];
18955
18956 let mut runner = Runner {
18957 state,
18958 roles: ResolvedRoles {
18959 implementers: Vec::new(),
18960 judges: Vec::new(),
18961 reviewers: Vec::new(),
18962 fixer: None,
18963 conductor: conductor(),
18964 implementer_roster: Vec::new(),
18965 judge_roster: Vec::new(),
18966 reviewer_roster: Vec::new(),
18967 },
18968 sem: Arc::new(Semaphore::new(1)),
18969 pause: Pause::new(),
18970 interrupt: Pause::new(),
18971 };
18972
18973 let started_marker = repo.join("started.marker");
18974 let release_marker = repo.join("release.marker");
18975 let poller = tokio::spawn(async move {
18976 for _ in 0..100 {
18981 if started_marker.exists()
18982 && let Ok(s) = crate::run::RunState::load(&run_id)
18983 && let Some(a) = s.active.get("gate")
18984 {
18985 std::fs::write(&release_marker, b"go").expect("release marker");
18986 return Some(a.clone());
18987 }
18988 tokio::time::sleep(Duration::from_millis(50)).await;
18989 }
18990 None
18991 });
18992
18993 runner.gate().await.expect("gate");
18994 let captured = poller.await.expect("poller task");
18995 let captured = captured.expect(
18996 "the poller never saw a `gate` task entry in run.json while the command was \
18997 still blocked on its own release marker",
18998 );
18999
19000 assert_eq!(captured.task.as_deref(), Some("gate"));
19001 assert_eq!(captured.node, "gate");
19002 assert_eq!(captured.index, Some(1));
19003 assert_eq!(captured.total, Some(1));
19004 assert!(
19005 captured
19006 .command
19007 .as_deref()
19008 .is_some_and(|c| c.contains("started.marker")),
19009 "{captured:?}"
19010 );
19011
19012 assert!(
19013 runner.state.active.is_empty(),
19014 "the entry must be cleared once the command actually finished: {:?}",
19015 runner.state.active
19016 );
19017 assert!(runner.state.gate_ran);
19018 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
19019 }
19020
19021 #[tokio::test]
19024 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
19025 use crate::verdict::{Finding, ReviewVote, Severity};
19026 crate::run::pin_test_home();
19027 let tmp = tempfile::tempdir().expect("tempdir");
19028 let repo = tmp.path().join("repo");
19029 std::fs::create_dir_all(&repo).unwrap();
19030 init_repo(&repo);
19031
19032 for (severity, vote, expect) in [
19033 (Severity::Major, ReviewVote::Reject, true),
19034 (Severity::Minor, ReviewVote::Reject, false),
19035 (Severity::Major, ReviewVote::Approve, false),
19036 ] {
19037 let mut round = review_round(false, 1, 1, 1, false, true);
19038 round.reviews = vec![ReviewRecord {
19039 reviewer: 1,
19040 agent: "alpha".to_owned(),
19041 summary: String::new(),
19042 findings: vec![Finding {
19043 id: "R1-1-1".to_owned(),
19044 severity,
19045 file: None,
19046 line: None,
19047 title: "t".to_owned(),
19048 detail: String::new(),
19049 }],
19050 vote: Some(vote),
19051 failed: None,
19052 duration_ms: 0,
19053 attempts: 0,
19054 }];
19055 let mut state = RunState::new(
19056 repo.clone(),
19057 "main".to_owned(),
19058 "deadbeef".to_owned(),
19059 "task".to_owned(),
19060 Config::default(),
19061 );
19062 state.reviews = vec![round];
19063 let mut runner = Runner {
19064 state,
19065 roles: ResolvedRoles {
19066 implementers: Vec::new(),
19067 judges: Vec::new(),
19068 reviewers: Vec::new(),
19069 fixer: None,
19070 conductor: conductor(),
19071 implementer_roster: Vec::new(),
19072 judge_roster: Vec::new(),
19073 reviewer_roster: Vec::new(),
19074 },
19075 sem: Arc::new(Semaphore::new(1)),
19076 pause: Pause::new(),
19077 interrupt: Pause::new(),
19078 };
19079 let shell = runner.state.config.shell();
19080 runner
19081 .stop_reviewing("round budget spent", &shell, &repo)
19082 .await
19083 .expect("stop_reviewing");
19084 assert_eq!(runner.state.status, RunStatus::Gating);
19085 assert_eq!(
19086 runner.state.contested_handoff.is_some(),
19087 expect,
19088 "{severity:?} + {vote:?}"
19089 );
19090 }
19091 }
19092
19093 #[tokio::test]
19106 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
19107 crate::run::pin_test_home();
19108 let home = crate::run::home();
19109
19110 let tmp = tempfile::tempdir().expect("tempdir");
19111 let repo = tmp.path().join("repo");
19112 std::fs::create_dir_all(&repo).unwrap();
19113 init_repo(&repo);
19114 let head = crate::git::rev_parse(&repo, "HEAD")
19115 .await
19116 .expect("rev-parse");
19117 let cache_dir = tmp.path().join("target");
19120
19121 let mut config = Config::default();
19122 config.verify.e2e = vec![format!(
19123 "CARGO_TARGET_DIR='{}' test -f README.md",
19124 cache_dir.display()
19125 )];
19126 config.graph.review_rounds = 1;
19127 config.graph.timeout_verify = Some(2);
19130 config.graph.timeout_resource_wait = 2;
19131
19132 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19133 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19134 .expect("no io error acquiring directly")
19135 {
19136 crate::cache::AcquireOutcome::Acquired(g) => g,
19137 crate::cache::AcquireOutcome::Busy(b) => {
19138 panic!("expected the direct acquire to win the lease first: {b:?}")
19139 }
19140 };
19141
19142 let mut state = RunState::new(
19143 repo.clone(),
19144 "main".to_owned(),
19145 head.clone(),
19146 "task".to_owned(),
19147 config,
19148 );
19149 state.candidates = vec![Candidate {
19150 index: 0,
19151 label: 'A',
19152 agent: "alpha".to_owned(),
19153 branch: "does-not-exist".to_owned(),
19154 worktree: repo.clone(),
19155 summary: String::new(),
19156 stat: String::new(),
19157 files: 0,
19158 commits: 0,
19159 empty: false,
19160 failed: None,
19161 verified_noop: None,
19162 duration_ms: 0,
19163 folded: false,
19164 }];
19165 state.tally = Some(Tally {
19166 first_choice: BTreeMap::from([('A', 1)]),
19167 borda: BTreeMap::new(),
19168 winner: 'A',
19169 rankings: 1,
19170 unanimous_initial: true,
19171 deliberated: false,
19172 changed_votes: 0,
19173 unanimous_final: true,
19174 tie_break: None,
19175 judges: 0,
19176 present: 0,
19177 quorum: 0,
19178 met_quorum: true,
19179 uncontested: Some("only candidate A produced a change".to_owned()),
19180 });
19181 state.reviews = vec![ReviewRound {
19185 round: 1,
19186 head: head.clone(),
19187 verified_head: None,
19188 verified_at: None,
19189 reviews: Vec::new(),
19190 e2e: Vec::new(),
19191 fix: None,
19192 blocking: 1,
19193 answered: 1,
19194 expected: 1,
19195 clean: false,
19196 verify_retried: false,
19197 e2e_deferred: true,
19198 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
19199 progressed: false,
19200 vote_split: false,
19201 reconsideration: Vec::new(),
19202 verdict: None,
19203 }];
19204
19205 let mut runner = Runner {
19206 state,
19207 roles: ResolvedRoles {
19208 implementers: Vec::new(),
19209 judges: Vec::new(),
19210 reviewers: Vec::new(),
19211 fixer: None,
19212 conductor: conductor(),
19213 implementer_roster: Vec::new(),
19214 judge_roster: Vec::new(),
19215 reviewer_roster: Vec::new(),
19216 },
19217 sem: Arc::new(Semaphore::new(1)),
19218 pause: Pause::new(),
19219 interrupt: Pause::new(),
19220 };
19221
19222 let shell = runner.state.config.shell();
19223 runner
19224 .stop_reviewing("round budget spent", &shell, &repo)
19225 .await
19226 .expect("stop_reviewing");
19227
19228 let last = runner.state.reviews.last().expect("round record");
19229 assert_eq!(
19230 last.e2e_status(),
19231 E2eStatus::ResourceBlocked,
19232 "the shared cache is still held; the attempt must read as blocked, not deferred or \
19233 failed: {last:?}"
19234 );
19235 assert_eq!(
19236 last.verified_head.as_deref(),
19237 Some(head.as_str()),
19238 "which commit this attempt targeted is known even though nothing finished checking \
19239 it"
19240 );
19241 let first_attempt_at = last
19242 .verified_at
19243 .expect("when this attempt ran is known too");
19244 assert_ne!(
19245 runner.state.status,
19246 RunStatus::Blocked,
19247 "contention is evidence about the machine, not the patch — it must not settle the \
19248 run as blocked: {:?}",
19249 runner.state.status
19250 );
19251 assert!(
19252 !runner
19253 .state
19254 .events
19255 .iter()
19256 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
19257 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
19258 runner.state.events
19259 );
19260
19261 runner
19266 .stop_reviewing("round budget spent", &shell, &repo)
19267 .await
19268 .expect("stop_reviewing retry");
19269 assert_eq!(
19270 runner.state.reviews.len(),
19271 1,
19272 "no new round was started: {:?}",
19273 runner.state.reviews
19274 );
19275 let last = runner.state.reviews.last().expect("round record");
19276 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
19277 assert!(
19278 last.verified_at.expect("still known") > first_attempt_at,
19279 "a second reentry must be a fresh attempt, not a stale copy of the first"
19280 );
19281 assert_ne!(runner.state.status, RunStatus::Blocked);
19282
19283 held.release();
19284 }
19285
19286 #[cfg(unix)]
19287 #[tokio::test]
19288 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
19289 ask_test_home();
19290 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
19291 let tmp = tempfile::tempdir().unwrap();
19292 let repo = tmp.path().join("repo");
19293 std::fs::create_dir(&repo).unwrap();
19294 init_repo(&repo);
19295 let run_git = |args: &[&str]| {
19296 let out = std::process::Command::new("git")
19297 .args(args)
19298 .current_dir(&repo)
19299 .output()
19300 .unwrap();
19301 assert!(
19302 out.status.success(),
19303 "{args:?}: {}",
19304 String::from_utf8_lossy(&out.stderr)
19305 );
19306 };
19307 let origin = tmp.path().join("origin.git");
19308 run_git(&["init", "--bare", origin.to_str().unwrap()]);
19309 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
19310 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
19311 run_git(&["add", "spec.rs"]);
19312 run_git(&["commit", "-m", "initial spec"]);
19313 run_git(&["push", "origin", "main"]);
19314 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
19315 run_git(&["checkout", "-b", "fix/review"]);
19316 std::fs::write(
19317 repo.join("check.rs"),
19318 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
19319 )
19320 .unwrap();
19321 run_git(&["add", "check.rs"]);
19322 run_git(&["commit", "-m", "use spec"]);
19323 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
19324 run_git(&["checkout", "main"]);
19325 std::fs::write(
19326 repo.join("spec.rs"),
19327 "struct AgentSpec { max_turn_secs: u64 }\n",
19328 )
19329 .unwrap();
19330 run_git(&["add", "spec.rs"]);
19331 run_git(&["commit", "-m", "extend spec"]);
19332 run_git(&["push", "origin", "main"]);
19333 run_git(&["checkout", "fix/review"]);
19334
19335 let mut runner = runner_at(RunStatus::Reviewing);
19336 runner.state.repo = repo.clone();
19337 runner.state.base_commit = base;
19338 runner.state.pr_head = Some("fix/review".into());
19339 runner.state.config.graph.review_rounds = max_rounds;
19340 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
19341 runner.state.config.graph.sessions = false;
19342 runner.state.config.disk.min_free_bytes = 0;
19343 runner.state.config.verify.e2e =
19344 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
19345 runner.state.candidates = vec![Candidate {
19346 index: 0,
19347 label: 'A',
19348 agent: "fixture".into(),
19349 branch: "fix/review".into(),
19350 worktree: repo.clone(),
19351 summary: String::new(),
19352 stat: String::new(),
19353 files: 1,
19354 commits: 1,
19355 empty: false,
19356 failed: None,
19357 verified_noop: None,
19358 duration_ms: 0,
19359 folded: false,
19360 }];
19361 runner.state.tally = Some(Tally {
19362 first_choice: BTreeMap::new(),
19363 borda: BTreeMap::new(),
19364 winner: 'A',
19365 rankings: 0,
19366 unanimous_initial: true,
19367 deliberated: false,
19368 changed_votes: 0,
19369 unanimous_final: true,
19370 tie_break: None,
19371 judges: 0,
19372 present: 0,
19373 quorum: 0,
19374 met_quorum: true,
19375 uncontested: Some("review-only".into()),
19376 });
19377 let mut last = review_round(false, 0, 1, 1, false, false);
19378 last.head = reviewed.clone();
19379 last.e2e_deferred = true;
19380 last.e2e[0].resource_blocked = true;
19381 last.e2e[0].output_tail = "waiting for the shared build cache".into();
19382 if max_rounds == 1 {
19383 last.e2e.clear();
19384 }
19385 runner.state.reviews = vec![last];
19386 let calls = tmp.path().join("fix-calls");
19387 let mut fixer = spec("repair");
19388 fixer
19389 .env
19390 .insert("FIX_CALLS".into(), calls.display().to_string());
19391 fixer.command = vec!["sh".into(), "-c".into(), r#"
19392 echo fix >> "$FIX_CALLS"
19393 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
19394 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
19395 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
19396 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
19397 "#.into()];
19398 let mut reviewer = spec("reviewer");
19399 reviewer.command = vec![
19400 "sh".into(),
19401 "-c".into(),
19402 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
19403 .into(),
19404 ];
19405 runner.roles.fixer = Some(vec![fixer]);
19406 runner.roles.reviewers = vec![reviewer];
19407
19408 runner.sync_to_base().await.unwrap();
19409 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
19410 assert_ne!(rebased, reviewed);
19411 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
19412 if reentry {
19413 let shell = runner.state.config.shell();
19416 runner
19417 .stop_unresolved_review("deferred verification", &shell, &repo)
19418 .await
19419 .unwrap();
19420 assert!(runner.catchup_repairable());
19421 assert!(!calls.exists());
19422 runner.state = RunState::load(&runner.state.id).unwrap();
19423 assert!(runner.catchup_repairable());
19424 }
19425 runner.review_loop().await.unwrap();
19426 let first = &runner.state.reviews[0];
19427 assert_eq!(first.head, reviewed);
19428 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
19429 assert!(
19430 first.verify_retried,
19431 "a reproduced build failure gets exactly one retry: {:?}",
19432 first.e2e
19433 );
19434 assert!(first.e2e[0].output_tail.contains("E0063"));
19435 assert!(!runner.catchup_repairable());
19436 if max_rounds == 2 {
19437 assert_eq!(
19438 runner.state.status,
19439 RunStatus::Gating,
19440 "{}",
19441 crate::report::run(&runner.state)
19442 );
19443 assert_eq!(runner.state.reviews.len(), 2);
19444 assert!(first.fix.as_ref().unwrap().committed);
19445 assert!(runner.state.reviews[1].clean);
19446 assert_eq!(
19447 runner
19448 .state
19449 .reviews
19450 .iter()
19451 .filter(|r| r.fix.is_some())
19452 .count(),
19453 1
19454 );
19455 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
19456 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
19457 } else {
19458 assert_eq!(runner.state.status, RunStatus::Blocked);
19459 assert!(first.fix.is_none());
19460 assert!(!calls.exists());
19461 let recovery = crate::recovery::assess(
19462 "prior",
19463 Some(&runner.state),
19464 &runner.landing_base(),
19465 crate::recovery::Relation::Diverged,
19466 );
19467 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
19468 let reasons = recovery.reasons.join("\n");
19469 assert!(
19470 reasons.contains(&reviewed) && reasons.contains(&rebased),
19471 "{reasons}"
19472 );
19473 assert!(
19474 reasons.contains("base sync to")
19475 && reasons.contains("additional review is required"),
19476 "{reasons}"
19477 );
19478 }
19479 }
19480 }
19481
19482 #[tokio::test]
19494 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
19495 crate::run::pin_test_home();
19496 let home = crate::run::home();
19497
19498 let tmp = tempfile::tempdir().expect("tempdir");
19499 let repo = tmp.path().join("repo");
19500 std::fs::create_dir_all(&repo).unwrap();
19501 init_repo(&repo);
19502 let head = crate::git::rev_parse(&repo, "HEAD")
19503 .await
19504 .expect("rev-parse");
19505 let cache_dir = tmp.path().join("target");
19506
19507 let mut config = Config::default();
19508 config.verify.e2e = vec![format!(
19509 "CARGO_TARGET_DIR='{}' test -f README.md",
19510 cache_dir.display()
19511 )];
19512 config.graph.review_rounds = 1;
19513 config.graph.timeout_verify = Some(2);
19514 config.graph.timeout_resource_wait = 2;
19515
19516 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19517 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19518 .expect("no io error acquiring directly")
19519 {
19520 crate::cache::AcquireOutcome::Acquired(g) => g,
19521 crate::cache::AcquireOutcome::Busy(b) => {
19522 panic!("expected the direct acquire to win the lease first: {b:?}")
19523 }
19524 };
19525
19526 let mut state = RunState::new(
19527 repo.clone(),
19528 "main".to_owned(),
19529 head.clone(),
19530 "task".to_owned(),
19531 config,
19532 );
19533 state.candidates = vec![Candidate {
19534 index: 0,
19535 label: 'A',
19536 agent: "alpha".to_owned(),
19537 branch: "does-not-exist".to_owned(),
19538 worktree: repo.clone(),
19539 summary: String::new(),
19540 stat: String::new(),
19541 files: 0,
19542 commits: 0,
19543 empty: false,
19544 failed: None,
19545 verified_noop: None,
19546 duration_ms: 0,
19547 folded: false,
19548 }];
19549 state.tally = Some(Tally {
19550 first_choice: BTreeMap::from([('A', 1)]),
19551 borda: BTreeMap::new(),
19552 winner: 'A',
19553 rankings: 1,
19554 unanimous_initial: true,
19555 deliberated: false,
19556 changed_votes: 0,
19557 unanimous_final: true,
19558 tie_break: None,
19559 judges: 0,
19560 present: 0,
19561 quorum: 0,
19562 met_quorum: true,
19563 uncontested: Some("only candidate A produced a change".to_owned()),
19564 });
19565 state.reviews = vec![ReviewRound {
19569 round: 1,
19570 head: head.clone(),
19571 verified_head: Some(head.clone()),
19572 verified_at: Some(jiff::Timestamp::now()),
19573 reviews: Vec::new(),
19574 e2e: vec![CommandOutcome {
19575 command: format!(
19576 "CARGO_TARGET_DIR='{}' test -f README.md",
19577 cache_dir.display()
19578 ),
19579 code: None,
19580 output_tail: "waiting for the shared build cache".to_owned(),
19581 duration_ms: 0,
19582 resource_blocked: true,
19583 }],
19584 fix: None,
19585 blocking: 1,
19586 answered: 1,
19587 expected: 1,
19588 clean: false,
19589 verify_retried: false,
19590 e2e_deferred: false,
19591 e2e_defer_reason: None,
19592 progressed: false,
19593 vote_split: false,
19594 reconsideration: Vec::new(),
19595 verdict: None,
19596 }];
19597
19598 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
19599 let mut runner = Runner {
19600 state,
19601 roles: ResolvedRoles {
19602 implementers: Vec::new(),
19603 judges: Vec::new(),
19604 reviewers: Vec::new(),
19605 fixer: None,
19606 conductor: conductor(),
19607 implementer_roster: Vec::new(),
19608 judge_roster: Vec::new(),
19609 reviewer_roster: Vec::new(),
19610 },
19611 sem: Arc::new(Semaphore::new(1)),
19612 pause: Pause::new(),
19613 interrupt: Pause::new(),
19614 };
19615
19616 runner.review_loop().await.expect("review_loop");
19621
19622 assert_eq!(
19623 runner.state.reviews.len(),
19624 1,
19625 "no new round was started on top of the unresolved one: {:?}",
19626 runner.state.reviews
19627 );
19628 let last = &runner.state.reviews[0];
19629 assert_eq!(
19630 last.e2e_status(),
19631 E2eStatus::ResourceBlocked,
19632 "still contended: {last:?}"
19633 );
19634 assert!(
19635 last.verified_at.expect("still known") > first_attempt_at,
19636 "review_loop must have actually retried the check, not left it exactly as found"
19637 );
19638 assert_ne!(
19639 runner.state.status,
19640 RunStatus::Blocked,
19641 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
19642 runner.state.status
19643 );
19644
19645 held.release();
19646 }
19647
19648 #[tokio::test]
19649 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
19650 crate::run::pin_test_home();
19651 for status in [RunStatus::Blocked, RunStatus::Gating] {
19652 let mut runner = runner_at(status);
19653 runner.state.config.graph.land = true;
19654 runner.state.gate_ran = true;
19655 runner.state.merge = Some(MergeOutcome {
19656 mode: MergeMode::Pr,
19657 ok: false,
19658 detail: "the fixer produced no commit".into(),
19659 empty: false,
19660 });
19661 let url = "https://example.invalid/o/r/pull/1";
19662 runner.state.pr = Some(crate::run::PrRecord {
19663 url: url.into(),
19664 number: 1,
19665 state: "open".into(),
19666 checks: "red".into(),
19667 round: 1,
19668 rounds: 1,
19669 red_at_merge: Vec::new(),
19670 });
19671 runner.state.save().unwrap();
19672 runner.execute_graph().await.unwrap();
19676 assert_eq!(runner.state.status, RunStatus::Blocked);
19677 assert!(runner.state.land_sync_pending);
19678 assert!(runner.state.events.iter().any(|event|
19679 event.node == "land" && event.message == format!("watching {url}")));
19680 assert_eq!(
19681 runner.state.merge.as_ref().unwrap().detail,
19682 "the fixer produced no commit"
19683 );
19684 }
19685 }
19686
19687 #[tokio::test]
19688 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
19689 crate::run::pin_test_home();
19690 let tmp = tempfile::tempdir().expect("tempdir");
19691 let repo = tmp.path().join("repo");
19692 std::fs::create_dir_all(&repo).unwrap();
19693 init_repo(&repo);
19694
19695 let mut config = Config::default();
19696 config.merge.mode = MergeMode::Pr;
19697 config.graph.land = true;
19698 config.graph.land_approval = false;
19699
19700 let mut state = RunState::new(
19701 repo.clone(),
19702 "main".to_owned(),
19703 "deadbeef".to_owned(),
19704 "task".to_owned(),
19705 config,
19706 );
19707 state.candidates = vec![Candidate {
19708 index: 0,
19709 label: 'A',
19710 agent: "alpha".to_owned(),
19711 branch: "does-not-exist".to_owned(),
19712 worktree: repo.clone(),
19713 summary: String::new(),
19714 stat: String::new(),
19715 files: 0,
19716 commits: 0,
19717 empty: false,
19718 failed: None,
19719 verified_noop: None,
19720 duration_ms: 0,
19721 folded: false,
19722 }];
19723 state.tally = Some(Tally {
19724 first_choice: BTreeMap::from([('A', 1)]),
19725 borda: BTreeMap::new(),
19726 winner: 'A',
19727 rankings: 1,
19728 unanimous_initial: true,
19729 deliberated: false,
19730 changed_votes: 0,
19731 unanimous_final: true,
19732 tie_break: None,
19733 judges: 0,
19734 present: 0,
19735 quorum: 0,
19736 met_quorum: true,
19737 uncontested: Some("only candidate A produced a change".to_owned()),
19738 });
19739 state.reviews = vec![ReviewRound {
19740 round: 1,
19741 head: "deadbeef".to_owned(),
19742 verified_head: None,
19743 verified_at: None,
19744 reviews: Vec::new(),
19745 e2e: Vec::new(),
19746 fix: None,
19747 blocking: 0,
19748 answered: 0,
19749 expected: 0,
19750 clean: true,
19751 verify_retried: false,
19752 e2e_deferred: false,
19753 e2e_defer_reason: None,
19754 progressed: false,
19755 vote_split: false,
19756 reconsideration: Vec::new(),
19757 verdict: None,
19758 }];
19759 state.gate = vec![CommandOutcome {
19760 command: "test".to_owned(),
19761 code: Some(0),
19762 output_tail: String::new(),
19763 duration_ms: 0,
19764 resource_blocked: false,
19765 }];
19766 state.gate_ran = true;
19767 state.status = RunStatus::Landing;
19771 state.merge = Some(MergeOutcome {
19772 mode: MergeMode::Pr,
19773 ok: true,
19774 detail: "https://example.invalid/x/y/pull/1".to_owned(),
19775 empty: false,
19776 });
19777
19778 ask_test_home();
19782 state.save().expect("save landing run");
19785 let store = ask::Questions::open();
19786 let q = ask_open_question(&store, &state.id);
19787
19788 let mut runner = Runner {
19789 state,
19790 roles: ResolvedRoles {
19791 implementers: Vec::new(),
19792 judges: Vec::new(),
19793 reviewers: Vec::new(),
19794 fixer: None,
19795 conductor: conductor(),
19796 implementer_roster: Vec::new(),
19797 judge_roster: Vec::new(),
19798 reviewer_roster: Vec::new(),
19799 },
19800 sem: Arc::new(Semaphore::new(1)),
19801 pause: Pause::new(),
19802 interrupt: Pause::new(),
19803 };
19804
19805 runner.execute().await.expect("execute");
19810
19811 assert_eq!(
19812 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19813 Some("https://example.invalid/x/y/pull/1"),
19814 "reentry must not push again or open a second pull request over the \
19815 one `land` is already watching"
19816 );
19817 assert_ne!(
19818 runner.state.status,
19819 RunStatus::Landing,
19820 "land could not actually reach the fake pull request, so it must \
19821 have given up rather than left the run silently parked forever"
19822 );
19823 assert_eq!(runner.state.status, RunStatus::Blocked);
19827 assert!(
19828 store.get(&q.id).unwrap().status.open(),
19829 "Blocked is still alive; settle_questions must have been a no-op here"
19830 );
19831 }
19832
19833 fn state_with_round(round: ReviewRound) -> RunState {
19834 let mut s = RunState::new(
19835 PathBuf::from("/repo"),
19836 "main".to_owned(),
19837 "abc1234".to_owned(),
19838 "add retries".to_owned(),
19839 Config::default(),
19840 );
19841 s.reviews = vec![round];
19842 s
19843 }
19844
19845 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19846 crate::verdict::Finding {
19847 id: id.to_owned(),
19848 severity,
19849 file: None,
19850 line: None,
19851 title: title.to_owned(),
19852 detail: String::new(),
19853 }
19854 }
19855
19856 #[test]
19857 fn pr_body_names_open_findings_and_declined_ones() {
19858 let round = ReviewRound {
19859 round: 2,
19860 head: "deadbee".to_owned(),
19861 verified_head: None,
19862 verified_at: None,
19863 reviews: vec![ReviewRecord {
19864 attempts: 0,
19865 reviewer: 1,
19866 agent: "alpha".to_owned(),
19867 summary: String::new(),
19868 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19869 vote: None,
19870 failed: None,
19871 duration_ms: 0,
19872 }],
19873 e2e: vec![CommandOutcome {
19874 command: "cargo test".to_owned(),
19875 code: Some(0),
19876 output_tail: String::new(),
19877 duration_ms: 0,
19878 resource_blocked: false,
19879 }],
19880 verify_retried: false,
19881 e2e_deferred: false,
19882 e2e_defer_reason: None,
19883 fix: Some(FixRecord {
19884 agent: "alpha".to_owned(),
19885 addressed: Vec::new(),
19886 rejected: vec![crate::verdict::Rejection {
19887 id: "R1-1-1".to_owned(),
19888 why: "not reachable from any caller".to_owned(),
19889 }],
19890 notes: String::new(),
19891 committed: true,
19892 failed: None,
19893 duration_ms: 0,
19894 continuation: None,
19895 }),
19896 blocking: 0,
19897 answered: 1,
19898 expected: 1,
19899 clean: false,
19900 progressed: true,
19901 vote_split: false,
19902 reconsideration: Vec::new(),
19903 verdict: None,
19904 };
19905 let state = state_with_round(round);
19906 let body = pr_message(&state, 'A').body;
19907
19908 assert!(body.contains("add retries"), "the task must still be there");
19909 assert!(body.contains("R2-1-1"), "{body}");
19910 assert!(!body.contains("unused import"), "{body}");
19911 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19912 assert!(
19913 !body.contains("not reachable from any caller"),
19914 "the reason it was declined: {body}"
19915 );
19916 }
19917
19918 #[test]
19919 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19920 let round = ReviewRound {
19921 round: 1,
19922 head: "deadbee".to_owned(),
19923 verified_head: None,
19924 verified_at: None,
19925 reviews: vec![ReviewRecord {
19926 attempts: 0,
19927 reviewer: 1,
19928 agent: "alpha".to_owned(),
19929 summary: String::new(),
19930 findings: Vec::new(),
19931 vote: None,
19932 failed: None,
19933 duration_ms: 0,
19934 }],
19935 e2e: Vec::new(),
19936 verify_retried: false,
19937 e2e_deferred: false,
19938 e2e_defer_reason: None,
19939 fix: None,
19940 blocking: 0,
19941 answered: 1,
19942 expected: 1,
19943 clean: true,
19944 progressed: false,
19945 vote_split: false,
19946 reconsideration: Vec::new(),
19947 verdict: None,
19948 };
19949 let state = state_with_round(round);
19950 let body = pr_message(&state, 'A').body;
19951 assert!(!body.contains("Open review findings"), "{body}");
19952 assert!(!body.contains("Declined"), "{body}");
19953 }
19954
19955 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19956 let mut state = RunState::new(
19957 PathBuf::from("/repo"),
19958 "main".to_owned(),
19959 "abc1234".to_owned(),
19960 instruction.to_owned(),
19961 Config::default(),
19962 );
19963 state.candidates.push(Candidate {
19964 index: 0,
19965 label: 'A',
19966 agent: "alpha".to_owned(),
19967 branch: "magi/x/A".to_owned(),
19968 worktree: PathBuf::from("/wt"),
19969 summary: summary.to_owned(),
19970 stat: String::new(),
19971 files: 1,
19972 commits: 1,
19973 empty: false,
19974 failed: None,
19975 verified_noop: None,
19976 folded: false,
19977 duration_ms: 0,
19978 });
19979 state
19980 }
19981
19982 fn review_state(subjects: &[&str]) -> RunState {
19983 let mut state = state_with_summary(
19984 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19985 "",
19986 );
19987 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19988 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19989 state
19990 }
19991
19992 #[tokio::test]
19995 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19996 ask_test_home();
19997 let tmp = tempfile::tempdir().unwrap();
19998 let repo = tmp.path().join("repo");
19999 std::fs::create_dir_all(&repo).unwrap();
20000 init_repo(&repo);
20001 let origin = tmp.path().join("origin.git");
20002 let g = |dir: &Path, args: &[&str]| {
20003 let out = std::process::Command::new("git")
20004 .args(args)
20005 .current_dir(dir)
20006 .quiet()
20007 .output()
20008 .expect("spawn git");
20009 assert!(
20010 out.status.success(),
20011 "git {args:?}: {}",
20012 String::from_utf8_lossy(&out.stderr)
20013 );
20014 };
20015 g(
20016 tmp.path(),
20017 &[
20018 "clone",
20019 "--bare",
20020 "-q",
20021 repo.to_str().unwrap(),
20022 origin.to_str().unwrap(),
20023 ],
20024 );
20025 g(
20026 &repo,
20027 &["remote", "add", "origin", origin.to_str().unwrap()],
20028 );
20029 let c1 = git::rev_parse(&repo, "main").await.unwrap();
20030
20031 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
20033 g(&repo, &["add", "-A"]);
20034 g(
20035 &repo,
20036 &["commit", "-q", "-m", "chore(deps): update a crate"],
20037 );
20038 g(&repo, &["push", "-q", "origin", "main"]);
20039 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
20040 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
20041 g(&repo, &["add", "-A"]);
20042 g(
20043 &repo,
20044 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
20045 );
20046 g(&repo, &["checkout", "-q", "main"]);
20047
20048 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
20049 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
20050 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
20052 assert_eq!(fallback, c1);
20053
20054 let mut state = review_state(&[
20055 "chore(deps): update a crate",
20056 "fix(daemon): apply a chosen action",
20057 ]);
20058 state.repo = repo.clone();
20059 state.base_branch = "main".to_owned();
20060 state.base_commit = c1;
20061 refresh_reviewed_commits(&mut state, "feat/own").await;
20062 assert_eq!(
20063 state.reviewed_commits,
20064 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
20065 );
20066 assert_eq!(
20067 review_title(&state).as_deref(),
20068 Some("fix(daemon): apply a chosen action")
20069 );
20070 assert_eq!(
20071 leaked_subjects(&state, "feat/own").await,
20072 Some(vec!["chore(deps): update a crate".to_owned()])
20073 );
20074 state.config.merge.remote = "nowhere".to_owned();
20077 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
20078 }
20079
20080 struct SummaryFixture {
20084 _tmp: tempfile::TempDir,
20085 repo: PathBuf,
20086 reply: PathBuf,
20087 runner: Runner,
20088 }
20089
20090 fn summary_fixture() -> SummaryFixture {
20091 ask_test_home();
20092 let tmp = tempfile::tempdir().unwrap();
20093 let repo = tmp.path().join("repo");
20094 std::fs::create_dir_all(&repo).unwrap();
20095 init_repo(&repo);
20096 let origin = tmp.path().join("origin.git");
20097 let g = |args: &[&str]| {
20098 let out = std::process::Command::new("git")
20099 .args(args)
20100 .current_dir(&repo)
20101 .quiet()
20102 .output()
20103 .expect("spawn git");
20104 assert!(
20105 out.status.success(),
20106 "git {args:?}: {}",
20107 String::from_utf8_lossy(&out.stderr)
20108 );
20109 };
20110 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
20111 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
20112 g(&["push", "-q", "origin", "main"]);
20113 g(&["checkout", "-q", "-b", "feat/own"]);
20114 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
20115 g(&["add", "-A"]);
20116 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
20117 let reply = tmp.path().join("reply.txt");
20118 std::fs::write(
20119 &reply,
20120 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
20121 )
20122 .unwrap();
20123 let mut runner = runner_at(RunStatus::Gating);
20124 let mut state = review_state(&["feat(own): add the own file"]);
20125 state.repo = repo.clone();
20126 state.base_branch = "main".to_owned();
20127 state.candidates[0].branch = "feat/own".to_owned();
20128 state.candidates[0].worktree = repo.clone();
20129 let mut author = spec("alpha");
20130 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
20131 author
20132 .env
20133 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
20134 state.config.agents = vec![author];
20135 runner.state = state;
20136 SummaryFixture {
20137 _tmp: tmp,
20138 repo,
20139 reply,
20140 runner,
20141 }
20142 }
20143
20144 async fn posted(f: &mut SummaryFixture) -> PrMessage {
20145 let winner = f.runner.state.candidates[0].clone();
20146 f.runner
20147 .guarded_pr_message(&winner, None, true)
20148 .await
20149 .unwrap()
20150 .expect("an English description never parks")
20151 }
20152
20153 fn clean_round_with_minor_findings() -> ReviewRound {
20154 let mut round = passing_round(2, None);
20155 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
20156 round.reviews = vec![ReviewRecord {
20157 attempts: 0,
20158 reviewer: 1,
20159 agent: "alpha".into(),
20160 summary: "内部レビュー".into(),
20161 findings: vec![
20162 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
20163 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
20164 ],
20165 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
20166 failed: None,
20167 duration_ms: 0,
20168 }];
20169 round
20170 }
20171
20172 #[tokio::test]
20173 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
20174 for review_only in [false, true] {
20175 let mut f = summary_fixture();
20176 if !review_only {
20177 f.runner.state.candidates[0].agent = "alpha".into();
20178 }
20179 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
20180 assert!(f.runner.state.reviews[0].clean);
20181 assert!(f.runner.state.reviews[0].fix.is_none());
20182 assert!(f.runner.state.open_findings().is_empty());
20183 assert_eq!(
20184 crate::pr_outcome::required_finding_ids(&f.runner.state),
20185 ["R2-1-1", "R2-1-2"]
20186 );
20187 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20188 let evidence = crate::pr_outcome::summary_evidence(
20189 &f.runner.state,
20190 crate::pr_outcome::FinalHead::At(&head),
20191 );
20192 assert!(
20193 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
20194 );
20195 let omitted = posted(&mut f).await;
20197 assert!(omitted.body.contains("No description"));
20198 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
20199 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
20200 assert!(f.runner.state.change_summary.is_none());
20201 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
20202 let described = posted(&mut f).await;
20203 assert!(described.body.contains("improve documentation for callers"));
20204 assert!(described.body.contains("remaining edge case"));
20205 assert!(described.body.contains("## Open review findings"));
20206 assert!(!described.body.contains("未修正"));
20207 }
20208 }
20209
20210 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
20211 let mut f = summary_fixture();
20212 if !review_only {
20213 f.runner.state.candidates[0].agent = "alpha".into();
20214 }
20215 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20216 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
20217 fixed
20218 .fix
20219 .as_mut()
20220 .unwrap()
20221 .rejected
20222 .push(crate::verdict::Rejection {
20223 id: "R1-1-1".into(),
20224 why: "呼び出されないコードのため辞退".into(),
20225 });
20226 let mut clean = clean_round_with_minor_findings();
20227 clean.verified_head = Some(head.clone());
20228 f.runner.state.reviews = vec![fixed, clean];
20229 ok_gate(&mut f.runner.state, Some(&head));
20230 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
20231 f.runner.state.config.agents.push(scripted("beta", reply));
20232 f.runner.state.config.roles.language_judge =
20233 Some(crate::config::AgentChoice::One("beta".into()));
20234 f
20235 }
20236
20237 #[tokio::test]
20238 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
20239 for review_only in [false, true] {
20240 let mut f = rewrite_fixture_with_findings(review_only,
20241 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
20242 let m = posted(&mut f).await;
20243 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20244 let expected =
20245 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
20246 assert!(m.body.contains(&expected));
20247 assert!(m.body.contains("## Review rounds"));
20248 assert!(
20249 m.body
20250 .contains("Gate (`verify.gate`): passed on the final commit")
20251 );
20252 assert!(
20253 m.body
20254 .contains("Tests (`verify.e2e`): passed on the final commit")
20255 );
20256 let c = f.runner.state.change_summary.as_ref().unwrap();
20257 assert!(
20258 m.body
20259 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
20260 );
20261 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20262 assert!(crate::pr_outcome::covers_required_findings(
20263 &f.runner.state,
20264 &m.body
20265 ));
20266 assert!(m.body.contains("documentation remains incomplete"));
20267 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
20268 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
20269 assert!(f.runner.state.github_text.is_none());
20270 }
20271 }
20272
20273 #[tokio::test]
20274 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
20275 for review_only in [false, true] {
20276 for reply in [
20277 "TITLE: fix: public description\nA short English description.",
20278 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
20279 ] {
20280 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
20281 let winner = f.runner.state.candidates[0].clone();
20282 let posted = f
20283 .runner
20284 .guarded_pr_message(&winner, None, true)
20285 .await
20286 .unwrap();
20287 assert!(posted.is_none());
20288 assert!(f.runner.state.parked);
20289 let withheld = f.runner.state.github_text.as_ref().unwrap();
20290 assert!(
20291 withheld.rewrite.is_none(),
20292 "an incomplete rewrite must not be selectable"
20293 );
20294 let q = ask::Questions::open()
20295 .get(withheld.question.as_ref().unwrap())
20296 .unwrap();
20297 assert!(
20298 !q.choices
20299 .contains(&crate::github_text::USE_REWRITE.to_owned())
20300 );
20301 assert!(
20302 q.detail.contains("R1-1-1")
20303 && q.detail.contains("R2-1-1")
20304 && q.detail.contains("R2-1-2")
20305 );
20306 }
20307 }
20308 }
20309
20310 #[tokio::test]
20311 async fn japanese_reports_stay_local_in_both_public_run_modes() {
20312 for review_only in [false, true] {
20313 let mut f = summary_fixture();
20314 if !review_only {
20315 f.runner.state.candidates[0].agent = "alpha".into();
20316 }
20317 f.runner.state.instruction = "内部タスクの説明".into();
20318 f.runner.state.candidates[0].summary = "内部の候補報告".into();
20319 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20320 f.runner.state.reviews = vec![
20321 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
20322 round_with_notes(2, Some("二回目の内部修正報告です。")),
20323 passing_round(3, Some(&head)),
20324 ];
20325 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
20326 ok_gate(&mut f.runner.state, Some(&head));
20327 let captured = f.repo.join("public-prompt.txt");
20328 let author = &mut f.runner.state.config.agents[0];
20329 author.command = vec![
20330 "sh".into(),
20331 "-c".into(),
20332 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20333 ];
20334 author.env.insert(
20335 "PROMPT_FILE".into(),
20336 captured.to_string_lossy().into_owned(),
20337 );
20338 let m = posted(&mut f).await;
20339 assert!(m.body.contains("Adds `own.txt`"));
20340 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
20341 assert!(m.body.contains("R1-1-1"));
20342 assert!(
20343 m.body
20344 .contains("Gate (`verify.gate`): passed on the final commit")
20345 );
20346 assert!(
20347 m.body
20348 .contains("Tests (`verify.e2e`): passed on the final commit")
20349 );
20350 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20351 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
20352 let prompt = std::fs::read_to_string(captured).unwrap();
20353 assert!(prompt.contains(&format!(
20354 "git diff {} {head}",
20355 f.runner.state.change_summary.as_ref().unwrap().base
20356 )));
20357 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
20358 assert_eq!(
20359 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
20360 "二回目の内部修正報告です。"
20361 );
20362 }
20363 }
20364
20365 #[tokio::test]
20366 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
20367 for review_only in [false, true] {
20368 let mut f = summary_fixture();
20369 if !review_only {
20370 f.runner.state.candidates[0].agent = "alpha".into();
20371 }
20372 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
20373 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
20374 std::fs::write(&f.reply, "no usable description").unwrap();
20375 let failed = posted(&mut f).await;
20376 assert!(
20377 failed
20378 .body
20379 .contains("No description of what this change does could be generated")
20380 );
20381 assert!(failed.body.is_ascii());
20382 assert!(f.runner.state.github_text.is_none());
20383 assert!(f.runner.state.change_summary.is_none());
20384 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20385 let evidence = crate::pr_outcome::summary_evidence(
20386 &f.runner.state,
20387 crate::pr_outcome::FinalHead::At(&head),
20388 );
20389 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
20390 assert!(
20391 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
20392 .exists()
20393 );
20394 f.runner.state =
20395 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
20396 std::fs::write(
20397 &f.reply,
20398 "TITLE: feat(own): add the own file\nAdds the committed own file.",
20399 )
20400 .unwrap();
20401 let retried = posted(&mut f).await;
20402 assert!(retried.body.contains("Adds the committed own file."));
20403 assert_eq!(
20404 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
20405 "日本語の修正報告"
20406 );
20407 }
20408 }
20409
20410 #[tokio::test]
20411 async fn same_head_verification_and_findings_invalidate_public_prose() {
20412 let mut f = summary_fixture();
20413 posted(&mut f).await;
20414 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20415 let original = f
20416 .runner
20417 .state
20418 .change_summary
20419 .as_ref()
20420 .unwrap()
20421 .evidence
20422 .clone();
20423 ok_gate(&mut f.runner.state, Some(&head));
20424 std::fs::write(
20425 &f.reply,
20426 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
20427 )
20428 .unwrap();
20429 let verified = posted(&mut f).await;
20430 assert!(verified.body.contains("now verified"));
20431 assert_ne!(
20432 f.runner.state.change_summary.as_ref().unwrap().evidence,
20433 original
20434 );
20435 let mut round = round_with_notes(1, Some("内部報告"));
20436 round.reviews.push(ReviewRecord {
20437 attempts: 0,
20438 reviewer: 1,
20439 agent: "alpha".into(),
20440 summary: "内部レビュー".into(),
20441 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
20442 vote: None,
20443 failed: None,
20444 duration_ms: 0,
20445 });
20446 round
20447 .fix
20448 .as_mut()
20449 .unwrap()
20450 .rejected
20451 .push(crate::verdict::Rejection {
20452 id: "R1-1-1".into(),
20453 why: "呼び出し元から到達できない".into(),
20454 });
20455 f.runner.state.reviews = vec![round];
20456 let omitted = posted(&mut f).await;
20458 assert!(omitted.body.contains("No description"));
20459 assert!(omitted.body.contains("R1-1-1"));
20460 assert!(!omitted.body.contains("到達"));
20461 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
20462 let described = posted(&mut f).await;
20463 assert!(described.body.contains("unreachable from callers"));
20464 assert!(described.body.contains("R1-1-1"));
20465 assert!(!described.body.contains("到達"));
20466 let evidence = f
20467 .runner
20468 .state
20469 .change_summary
20470 .as_ref()
20471 .unwrap()
20472 .evidence
20473 .clone();
20474 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
20475 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
20476 assert!(posted(&mut f).await.body.contains("rationale updated"));
20477 assert_ne!(
20478 f.runner.state.change_summary.as_ref().unwrap().evidence,
20479 evidence
20480 );
20481 f.runner
20483 .state
20484 .change_summary
20485 .as_mut()
20486 .unwrap()
20487 .evidence
20488 .clear();
20489 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
20490 assert!(posted(&mut f).await.body.contains("current specification"));
20491 }
20492
20493 #[tokio::test]
20494 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
20495 let mut f = summary_fixture();
20496 let m = posted(&mut f).await;
20497 assert_eq!(m.title, "feat(own): add the own file");
20498 assert!(
20499 m.body
20500 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
20501 "{}",
20502 m.body
20503 );
20504 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
20505 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
20506 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20507 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20508 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
20509 let kept = f.runner.state.change_summary.as_ref().unwrap();
20510 assert_eq!(
20511 (kept.head.as_str(), kept.base.as_str()),
20512 (head.as_str(), base.as_str())
20513 );
20514 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
20515 }
20516
20517 #[tokio::test]
20518 async fn review_only_summary_receives_the_configured_github_language() {
20519 let mut f = summary_fixture();
20520 f.runner.state.config.graph.github_language = "ja".into();
20521 let captured = f.repo.join("summary-prompt.txt");
20522 let author = &mut f.runner.state.config.agents[0];
20523 author.command = vec![
20524 "sh".into(),
20525 "-c".into(),
20526 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20527 ];
20528 author.env.insert(
20529 "PROMPT_FILE".into(),
20530 captured.to_string_lossy().into_owned(),
20531 );
20532 std::fs::write(
20533 &f.reply,
20534 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
20535 )
20536 .unwrap();
20537 let m = posted(&mut f).await;
20538 let prompt = std::fs::read_to_string(captured).unwrap();
20539 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
20540 assert!(!prompt.contains("in English"));
20541 assert_eq!(m.title, "feat(own): ファイルを追加");
20542 assert!(m.body.contains("own.txt を追加します。"));
20543 }
20544
20545 #[tokio::test]
20546 async fn review_fixes_come_after_the_whole_change_summary() {
20547 let mut f = summary_fixture();
20548 f.runner.state.reviews = vec![
20549 round_with_notes(1, Some("handled the PATH case")),
20550 round_with_notes(2, None),
20551 ];
20552 let m = posted(&mut f).await;
20553 let summary = m.body.find("## Summary").expect("summary");
20554 let fixes = m.body.find("## Review rounds").expect("review rounds");
20555 assert!(summary < fixes, "{}", m.body);
20556 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
20557 assert!(m.body.contains("## Final verification"), "{}", m.body);
20558 }
20559
20560 #[tokio::test]
20561 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
20562 let mut f = summary_fixture();
20563 let first = posted(&mut f).await;
20564 let first_head = f.runner.state.change_summary.clone().unwrap().head;
20565 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
20567 let again = posted(&mut f).await;
20568 assert_eq!((again.title, again.body), (first.title, first.body));
20569
20570 let g = |args: &[&str]| {
20571 let out = std::process::Command::new("git")
20572 .args(args)
20573 .current_dir(&f.repo)
20574 .quiet()
20575 .output()
20576 .expect("spawn git");
20577 assert!(out.status.success(), "git {args:?}");
20578 };
20579 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
20580 g(&["add", "-A"]);
20581 g(&["commit", "-q", "-m", "fix(own): handle more"]);
20582 let second = posted(&mut f).await;
20583 let kept = f.runner.state.change_summary.clone().unwrap();
20584 assert_ne!(kept.head, first_head);
20585 assert!(second.body.contains("Not read yet."), "{}", second.body);
20586 assert!(!second.body.contains("so the first summary is read"));
20587 assert_eq!(second.title, "feat(own): other");
20588 }
20589
20590 #[tokio::test]
20591 async fn an_unavailable_summary_is_stated_and_never_invented() {
20592 let mut f = summary_fixture();
20593 std::fs::write(&f.reply, "no title line here").unwrap();
20594 let m = posted(&mut f).await;
20595 assert!(
20596 m.body
20597 .contains("No description of what this change does could be generated"),
20598 "{}",
20599 m.body
20600 );
20601 assert!(m.body.contains("internal reports remain in local records"));
20602 assert!(f.runner.state.change_summary.is_none());
20603 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
20604 let winner = f.runner.state.candidates[0].clone();
20606 let preview = f
20607 .runner
20608 .guarded_pr_message(&winner, None, false)
20609 .await
20610 .unwrap()
20611 .unwrap();
20612 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
20613 }
20614
20615 #[test]
20616 fn pr_message_review_single_commit_uses_its_subject() {
20617 let state = review_state(&["feat(nats): per-role user"]);
20618 let m = pr_message(&state, 'A');
20619 assert_eq!(m.title, "feat(nats): per-role user");
20620 assert!(!m.body.contains("Review the work already"), "{}", m.body);
20621 assert!(m.body.contains("## Commits under review"), "{}", m.body);
20622 }
20623
20624 #[test]
20625 fn pr_message_review_multi_commit_takes_the_oldest() {
20626 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
20627 let m = pr_message(&state, 'A');
20628 assert_eq!(m.title, "feat: the change");
20629 for s in ["feat: the change", "fix: typo", "fix: again"] {
20630 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
20631 }
20632 }
20633
20634 #[test]
20635 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
20636 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
20637 let mut state = review_state(&[first, "fix: later fixup"]);
20638 state.candidates[0].branch = "機能/ログイン".to_owned();
20639 let m = pr_message(&state, 'A');
20640 assert!(
20641 m.title.starts_with("chore: land candidate A of run"),
20642 "{}",
20643 m.title
20644 );
20645 }
20646 }
20647
20648 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
20649 BranchFacts {
20650 commits: commits
20651 .iter()
20652 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
20653 .collect(),
20654 stat: stat.to_owned(),
20655 }
20656 }
20657
20658 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
20659 let mut r = review_round(true, 0, 1, 1, true, true);
20660 r.round = round;
20661 r.fix = notes.map(|n| FixRecord {
20662 agent: "fixer".to_owned(),
20663 addressed: Vec::new(),
20664 rejected: Vec::new(),
20665 notes: n.to_owned(),
20666 committed: true,
20667 failed: None,
20668 duration_ms: 0,
20669 continuation: None,
20670 });
20671 r
20672 }
20673
20674 #[test]
20675 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
20676 let state = review_state(&["ignored"]);
20677 let f = facts(
20678 &[
20679 (
20680 "fix(login): resolve PATH on macOS",
20681 "Login shells skip rc files.",
20682 ),
20683 ("fix: address review", ""),
20684 ],
20685 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
20686 );
20687 let m = pr_message_with(&state, 'A', Some(&f));
20688 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
20689 assert!(
20690 m.body
20691 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
20692 "{}",
20693 m.body
20694 );
20695 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
20696 assert!(m.body.contains("## Diff stat"), "{}", m.body);
20697 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
20698 for banned in [
20699 "Review the work already",
20700 "no task statement",
20701 "Original task",
20702 ] {
20703 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
20704 }
20705 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
20706 }
20707
20708 #[test]
20709 fn pr_message_review_truncates_a_huge_first_commit_body() {
20710 let state = review_state(&["ignored"]);
20711 let f = facts(
20712 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
20713 "s",
20714 );
20715 let m = pr_message_with(&state, 'A', Some(&f));
20716 assert!(m.body.len() < 30_000, "{}", m.body.len());
20717 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
20718 assert!(
20719 m.body.contains("- ... 1 more commit(s)"),
20720 "{}",
20721 m.body.len()
20722 );
20723 assert!(m.body.ends_with("magi:candidate-a\n"));
20724 }
20725
20726 #[test]
20727 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
20728 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
20729 assert_eq!(m.title, "feat: x");
20730 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
20731 assert!(!m.body.contains("Diff stat"), "{}", m.body);
20732 assert!(!m.body.contains("no task statement"), "{}", m.body);
20733 }
20734
20735 #[test]
20736 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
20737 let mut state = review_state(&["日本語の件名"]);
20738 state.candidates[0].branch = "fix/macos-login-path".to_owned();
20739 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
20740 state.candidates[0].branch = "機能/ログイン".to_owned();
20741 assert!(
20742 pr_message(&state, 'A')
20743 .title
20744 .starts_with("chore: land candidate A")
20745 );
20746 }
20747
20748 #[test]
20749 fn pr_message_review_fixes_survive_a_clean_final_round() {
20750 let mut state = review_state(&["feat: x"]);
20751 state.reviews = vec![
20752 round_with_notes(1, Some("handled the PATH case")),
20753 round_with_notes(2, None),
20754 ];
20755 let body = pr_message(&state, 'A').body;
20756 assert!(
20757 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
20758 "{body}"
20759 );
20760 assert!(body.contains("### Round 1"), "{body}");
20761
20762 state.reviews = vec![
20763 round_with_notes(1, Some("first fix")),
20764 round_with_notes(2, Some("")),
20765 round_with_notes(3, Some("second fix")),
20766 round_with_notes(4, None),
20767 ];
20768 let body = pr_message(&state, 'A').body;
20769 assert!(
20770 body.contains("### Round 1") && !body.contains("first fix"),
20771 "{body}"
20772 );
20773 assert!(
20774 body.contains("### Round 3") && !body.contains("second fix"),
20775 "{body}"
20776 );
20777 assert!(body.contains("### Round 2"), "{body}");
20778 }
20779
20780 const FINAL: &str = "abcdef1234567890";
20781
20782 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20783 let mut r = round_with_notes(round, None);
20784 r.verified_head = verified.map(str::to_owned);
20785 r
20786 }
20787
20788 fn ok_gate(state: &mut RunState, head: Option<&str>) {
20789 state.gate = vec![CommandOutcome {
20790 command: "gate".to_owned(),
20791 code: Some(0),
20792 output_tail: String::new(),
20793 duration_ms: 0,
20794 resource_blocked: false,
20795 }];
20796 state.gate_ran = true;
20797 state.gate_head = head.map(str::to_owned);
20798 }
20799
20800 fn body_at(state: &RunState) -> String {
20801 pr_message_raw(
20802 state,
20803 'A',
20804 None,
20805 crate::pr_outcome::FinalHead::At(FINAL),
20806 ChangeNote::NotAsked,
20807 )
20808 .body
20809 }
20810
20811 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20812
20813 #[test]
20814 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20815 let mut state = review_state(&["feat: x"]);
20816 state.reviews = vec![
20817 round_with_notes(1, Some(SANDBOXED)),
20818 passing_round(2, Some(FINAL)),
20819 ];
20820 ok_gate(&mut state, Some(FINAL));
20821 let body = body_at(&state);
20822 assert!(!body.contains("Handled the PATH case."), "{body}");
20823 assert!(!body.contains("could not commit"), "{body}");
20824 assert!(!body.contains("could not run"), "{body}");
20825 assert!(
20826 body.contains("The fix is committed on the branch"),
20827 "{body}"
20828 );
20829 assert!(!body.contains("sandbox"), "{body}");
20830 assert!(
20831 body.contains("Tests (`verify.e2e`): passed on the final commit"),
20832 "{body}"
20833 );
20834 assert!(
20835 body.contains("Gate (`verify.gate`): passed on the final commit"),
20836 "{body}"
20837 );
20838 assert!(
20840 state.reviews[0]
20841 .fix
20842 .as_ref()
20843 .unwrap()
20844 .notes
20845 .contains("could not commit")
20846 );
20847 }
20848
20849 #[test]
20850 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20851 let mut state = review_state(&["feat: x"]);
20852 let mut r1 = round_with_notes(1, Some(SANDBOXED));
20853 r1.fix.as_mut().unwrap().committed = false;
20854 let mut r2 = passing_round(2, Some(FINAL));
20855 r2.e2e[0].code = Some(1);
20856 state.reviews = vec![r1, r2];
20857 let body = body_at(&state);
20858 assert!(!body.contains("could not commit"), "{body}");
20859 assert!(!body.contains("could not run"), "{body}");
20860 assert!(body.contains("This round produced no commit"), "{body}");
20861 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20862 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20863 assert!(!body.contains("passed"), "{body}");
20864 }
20865
20866 #[test]
20867 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20868 let mut state = review_state(&["feat: x"]);
20869 state.reviews = vec![
20870 round_with_notes(1, Some(SANDBOXED)),
20871 passing_round(2, Some("0123456789abcdef")),
20872 ];
20873 ok_gate(&mut state, Some("0123456789abcdef"));
20874 let body = body_at(&state);
20875 assert!(!body.contains("HTTP tests could not run"), "{body}");
20876 assert!(!body.contains("could not commit"), "{body}");
20878 assert!(
20879 body.contains("passed on an earlier commit (`0123456`)"),
20880 "{body}"
20881 );
20882 assert!(!body.contains("passed on the final commit"), "{body}");
20883
20884 ok_gate(&mut state, None);
20886 let body = body_at(&state);
20887 assert!(
20888 body.contains("passed on a commit that was not recorded"),
20889 "{body}"
20890 );
20891 }
20892
20893 #[test]
20894 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20895 let state = state_with_summary(
20896 "add retries to the client",
20897 "TITLE: feat: retries\n\nDid it.",
20898 );
20899 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20900 assert_eq!(m.title, "feat: retries");
20901 assert!(
20902 m.body.contains("<summary>Original task</summary>"),
20903 "{}",
20904 m.body
20905 );
20906 assert!(!m.body.contains("Commits under review"), "{}", m.body);
20907 assert!(
20908 m.body
20909 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20910 "{}",
20911 m.body
20912 );
20913 }
20914
20915 #[test]
20916 fn pr_message_review_bounds_a_long_english_subject() {
20917 let long = format!("feat: {}", "word ".repeat(100));
20918 let m = pr_message(&review_state(&[&long]), 'A');
20919 assert!(m.title.starts_with("feat: word"), "{}", m.title);
20920 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20921 }
20922
20923 #[test]
20924 fn pr_message_implementation_run_is_unchanged_by_review_support() {
20925 let state = state_with_summary("add retries\n\ndetails", "- did some things");
20926 let m = pr_message(&state, 'A');
20927 assert_eq!(m.title, "add retries");
20928 assert!(m.body.contains("<summary>Original task</summary>"));
20929 assert!(!m.body.contains("Commits under review"));
20930 assert_eq!(landing_subject_source(&state), state.instruction);
20931 }
20932
20933 #[test]
20934 fn review_run_squash_subject_is_the_change_not_the_prompt() {
20935 let state = review_state(&["feat: the change", "fix: typo"]);
20936 let source = landing_subject_source(&state);
20937 assert_eq!(land::merge_subject("", &source), "feat: the change");
20938 assert_eq!(
20939 land::merge_subject("magi: candidate A (uncommitted work)", &source),
20940 "feat: the change"
20941 );
20942 assert_eq!(
20944 land::merge_subject("feat: renamed by hand", &source),
20945 "feat: renamed by hand"
20946 );
20947 let blank = review_state(&["日本語"]);
20948 assert!(
20949 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20950 );
20951 }
20952
20953 #[test]
20954 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20955 let state = review_state(&["feat: the change"]);
20956 let source = landing_subject_source(&state);
20957 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20958 assert_eq!(
20959 land::merge_subject(landing_title(&state, old), &source),
20960 "feat: the change"
20961 );
20962 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20963 let task = state_with_summary("add retries", "");
20964 assert_eq!(landing_title(&task, old), old);
20965 }
20966
20967 #[test]
20968 fn pr_message_describes_the_change_not_the_task() {
20969 let state = state_with_summary(
20970 "今回やってほしいこと: results projector を直す",
20971 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20972 );
20973 let m = pr_message(&state, 'A');
20974 assert_eq!(m.title, "fix(web): batch the runs list reads");
20975 assert!(
20976 m.body.starts_with("## Summary\n\n- reads run.json once"),
20977 "{}",
20978 m.body
20979 );
20980 assert!(!m.body.contains("TITLE:"), "{}", m.body);
20981 let task_at = m.body.find("今回やってほしいこと").unwrap();
20982 let details_at = m.body.find("<details>").unwrap();
20983 assert!(
20984 details_at < task_at,
20985 "the task lives inside <details>: {}",
20986 m.body
20987 );
20988 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20989 assert!(m.body.contains("magi:candidate-a"));
20990 }
20991
20992 #[test]
20993 fn pr_message_falls_back_to_the_task_without_a_title_line() {
20994 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20995 let m = pr_message(&state, 'A');
20996 assert_eq!(m.title, "add retries");
20997 assert!(
20998 m.body.contains("## Summary\n\n- did some things"),
20999 "{}",
21000 m.body
21001 );
21002
21003 let none = RunState::new(
21004 PathBuf::from("/repo"),
21005 "main".to_owned(),
21006 "abc1234".to_owned(),
21007 "add retries".to_owned(),
21008 Config::default(),
21009 );
21010 let m = pr_message(&none, 'A');
21011 assert_eq!(m.title, "add retries");
21012 assert!(!m.body.contains("## Summary"), "{}", m.body);
21013 }
21014
21015 #[test]
21016 fn pr_message_refuses_the_candidate_commit_subject() {
21017 for bad in [
21018 "TITLE: magi: candidate A (uncommitted work)",
21019 "TITLE: chore: stuff (uncommitted work)",
21020 "TITLE: ",
21021 ] {
21022 let state = state_with_summary("add retries", bad);
21023 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
21024 }
21025 }
21026
21027 #[test]
21028 fn pr_message_bounds_a_very_long_task_and_title() {
21029 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
21030 let state = state_with_summary(&long, "- nothing");
21031 let m = pr_message(&state, 'A');
21032 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21033 assert!(!m.title.contains('\n'));
21034
21035 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
21036 let m = pr_message(&state, 'A');
21037 assert!(m.title.starts_with("feat: "));
21038 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21039 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
21040 }
21041
21042 fn long_title_of(instruction: &str) -> String {
21043 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
21044 }
21045
21046 #[test]
21047 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
21048 let first = "Make the landing path keep a readable title for long tasks";
21049 let line = format!(
21050 "{first}. {}",
21051 "And then keep going with more words ".repeat(20)
21052 );
21053 let t = long_title_of(&line);
21054 assert_eq!(t, first);
21055 assert!(!t.starts_with("chore: land"));
21056 }
21057
21058 #[test]
21059 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
21060 let line = "word ".repeat(200);
21061 let t = long_title_of(&line);
21062 assert!(t.ends_with("word..."), "{t}");
21063 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
21064 }
21065
21066 #[test]
21067 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
21068 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
21069 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
21070 }
21071 }
21072
21073 #[test]
21074 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
21075 let mut state = state_with_summary("再試行を追加する", "- nothing");
21076 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
21077 state.config.graph.github_language = "ja".into();
21078 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
21079 }
21080
21081 #[test]
21082 fn pr_message_title_limit_is_exact() {
21083 let at = "a".repeat(PR_TITLE_MAX);
21084 assert_eq!(long_title_of(&at), at);
21085 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
21086 assert!(over.ends_with("..."), "{over}");
21087 assert_eq!(over.chars().count(), PR_TITLE_MAX);
21088 }
21089
21090 #[test]
21091 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
21092 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
21093 let t = long_title_of(&line);
21094 assert!(t.ends_with("..."), "{t}");
21095 assert!(t.is_ascii(), "{t}");
21096 }
21097
21098 #[test]
21099 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
21100 let line = format!(
21101 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
21102 "plus more filler words ".repeat(20)
21103 );
21104 let t = long_title_of(&line);
21105 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
21106 }
21107
21108 #[test]
21109 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
21110 let line = format!("\n\n# {}", "title words ".repeat(40));
21111 let state = state_with_summary(&line, "- nothing");
21112 let m = pr_message(&state, 'A');
21113 assert!(m.title.starts_with("title words"), "{}", m.title);
21114 assert_eq!(
21115 land::merge_subject(&m.title, &landing_subject_source(&state)),
21116 m.title
21117 );
21118 assert_eq!(
21120 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
21121 "feat: renamed by hand"
21122 );
21123 }
21124
21125 #[tokio::test]
21126 async fn github_text_rewrite_is_bounded_and_falls_back() {
21127 crate::run::pin_test_home();
21128 for (reply, accepted) in [
21129 (
21130 "TITLE: fix: retries\nAdd retries for failed requests.",
21131 true,
21132 ),
21133 (
21134 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
21135 false,
21136 ),
21137 (
21138 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
21139 true,
21140 ),
21141 (
21142 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
21143 true,
21144 ),
21145 (
21146 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
21147 true,
21148 ),
21149 (
21150 "TITLE: fix: retries\n日本語の説明です。 token=secret",
21151 false,
21152 ),
21153 ("TITLE: fix: retries\n", false),
21154 ] {
21155 let dir = tempfile::tempdir().unwrap();
21156 let mut runner = runner_at(RunStatus::Gating);
21157 runner.state = state_with_summary(
21158 "add retries",
21159 "TITLE: fix: retries\n日本語の説明を書きます。",
21160 );
21161 runner.state.repo = dir.path().to_owned();
21162 runner.state.candidates[0].worktree = dir.path().to_owned();
21163 let mut author = spec("alpha");
21164 author.command = vec![
21165 "sh".into(),
21166 "-c".into(),
21167 "printf '%s' \"$REWRITE_REPLY\"".into(),
21168 ];
21169 author.env.insert("REWRITE_REPLY".into(), reply.into());
21170 runner.state.config.agents = vec![author];
21171 seed_public_summary(&mut runner);
21172 let winner = runner.state.candidates[0].clone();
21173 let first = runner
21174 .guarded_pr_message(&winner, None, true)
21175 .await
21176 .unwrap();
21177 let message = if accepted {
21178 first.expect("a passing rewrite never asks")
21179 } else {
21180 assert!(first.is_none());
21182 assert!(runner.state.parked);
21183 let store = ask::Questions::open();
21184 let asked: Vec<_> = store
21185 .list()
21186 .into_iter()
21187 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21188 .collect();
21189 assert_eq!(asked.len(), 1);
21190 assert!(!asked[0].detail.contains("secret"));
21191 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
21192 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
21193 assert_eq!(
21194 asked[0]
21195 .choices
21196 .contains(&crate::github_text::USE_REWRITE.to_owned()),
21197 rewrite.is_some()
21198 );
21199 store
21202 .update(&asked[0].id, |q| {
21203 q.abandon("test");
21204 Ok(())
21205 })
21206 .unwrap();
21207 let again = runner
21208 .guarded_pr_message(&winner, None, true)
21209 .await
21210 .unwrap()
21211 .expect("an abandoned question falls back");
21212 let n = store
21213 .list()
21214 .into_iter()
21215 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21216 .count();
21217 assert_eq!(n, 1, "asked once per rejected text");
21218 again
21219 };
21220 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
21221 assert!(!message.body.contains("secret"));
21222 assert!(!message.body.contains("prefix"));
21223 assert!(!message.title.contains("secret"));
21224 assert_eq!(
21225 message.body.contains("Add retries for failed requests."),
21226 accepted
21227 );
21228 assert_eq!(
21229 runner.state.seats["github-text-rewrite"].turns, 1,
21230 "only one rewrite invocation"
21231 );
21232 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
21233 if accepted {
21234 assert!(runner.state.github_text.is_none());
21235 assert!(!runner.state.parked);
21236 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
21237 assert!(message.body.contains("## Final verification"));
21238 assert!(message.body.contains("Evidence: final diff"));
21239 assert!(
21240 message
21241 .body
21242 .contains(&format!("magi:run/{}", runner.state.id))
21243 );
21244 }
21245 if !accepted {
21246 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21247 assert!(
21248 message
21249 .body
21250 .contains(&format!("magi:run/{}", runner.state.id))
21251 );
21252 }
21253 }
21254 }
21255
21256 fn scripted(id: &str, reply: &str) -> AgentSpec {
21257 let mut a = spec(id);
21258 a.command = vec![
21259 "sh".into(),
21260 "-c".into(),
21261 "printf '%s' \"$REWRITE_REPLY\"".into(),
21262 ];
21263 a.env.insert("REWRITE_REPLY".into(), reply.into());
21264 a
21265 }
21266
21267 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
21268 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
21269
21270 fn seed_public_summary(runner: &mut Runner) {
21272 let repo = &runner.state.repo;
21273 let g = |args: &[&str]| {
21274 let out = std::process::Command::new("git")
21275 .args(args)
21276 .current_dir(repo)
21277 .quiet()
21278 .output()
21279 .unwrap();
21280 assert!(out.status.success(), "git {args:?}");
21281 String::from_utf8_lossy(&out.stdout).trim().to_owned()
21282 };
21283 g(&["init", "-q", "-b", "main"]);
21284 g(&[
21285 "-c",
21286 "user.name=Test",
21287 "-c",
21288 "user.email=test@example.invalid",
21289 "commit",
21290 "-q",
21291 "--allow-empty",
21292 "-m",
21293 "feat: fixture",
21294 ]);
21295 let head = g(&["rev-parse", "HEAD"]);
21296 g(&["update-ref", "refs/remotes/origin/main", &head]);
21297 let evidence = crate::pr_outcome::summary_evidence(
21298 &runner.state,
21299 crate::pr_outcome::FinalHead::At(&head),
21300 );
21301 let generated = &runner.state.candidates[0].summary;
21302 runner.state.change_summary = Some(crate::run::ChangeSummary {
21303 base: head.clone(),
21304 head,
21305 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
21306 title: summary_title(generated).unwrap(),
21307 body: summary_without_title(generated),
21308 });
21309 }
21310
21311 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
21312 let mut runner = runner_at(RunStatus::Gating);
21313 runner.state = state_with_summary("add retries", JA_SUMMARY);
21314 if review_only {
21315 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21316 }
21317 runner.state.repo = dir.to_owned();
21318 runner.state.candidates[0].worktree = dir.to_owned();
21319 runner.state.config.agents = agents;
21320 seed_public_summary(&mut runner);
21321 runner
21322 }
21323
21324 fn gate_questions(runner: &Runner) -> usize {
21325 ask::Questions::open()
21326 .list()
21327 .into_iter()
21328 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21329 .count()
21330 }
21331
21332 #[tokio::test]
21333 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
21334 crate::run::pin_test_home();
21335 let dir = tempfile::tempdir().unwrap();
21336 let mut runner = rewrite_runner(
21337 dir.path(),
21338 true,
21339 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21340 );
21341 runner.state.config.roles.language_judge =
21342 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
21343 let winner = runner.state.candidates[0].clone();
21344 let msg = runner
21345 .guarded_pr_message(&winner, None, true)
21346 .await
21347 .unwrap()
21348 .expect("an English rewrite posts without asking");
21349 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
21350 assert!(msg.body.contains("<details>kept</details>"));
21351 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
21352 assert_eq!(gate_questions(&runner), 0);
21353 assert!(runner.state.github_text.is_none());
21354 assert!(!runner.state.seats.contains_key("impl-A"));
21355 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21356 }
21357
21358 #[tokio::test]
21359 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
21360 crate::run::pin_test_home();
21361 let dir = tempfile::tempdir().unwrap();
21362 let mut runner = rewrite_runner(
21363 dir.path(),
21364 false,
21365 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21366 );
21367 runner.state.config.roles.language_judge =
21368 Some(crate::config::AgentChoice::One("beta".into()));
21369 runner.state.seats.insert(
21370 "impl-A".into(),
21371 SeatState::new("impl-A", "alpha", runner.state.seed),
21372 );
21373 let winner = runner.state.candidates[0].clone();
21374 let msg = runner
21375 .guarded_pr_message(&winner, None, true)
21376 .await
21377 .unwrap()
21378 .expect("rewritten");
21379 assert!(msg.body.contains("<details>kept</details>"));
21380 assert_eq!(runner.state.seats["impl-A"].turns, 0);
21381 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21382 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21383 }
21384
21385 #[tokio::test]
21386 async fn rewrite_reuses_a_resumable_implementer_session() {
21387 crate::run::pin_test_home();
21388 let dir = tempfile::tempdir().unwrap();
21389 let mut runner = rewrite_runner(
21390 dir.path(),
21391 false,
21392 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
21393 );
21394 runner.state.config.roles.language_judge =
21395 Some(crate::config::AgentChoice::One("beta".into()));
21396 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21397 seat.turns = 1;
21398 runner.state.seats.insert("impl-A".into(), seat);
21399 let winner = runner.state.candidates[0].clone();
21400 runner
21401 .guarded_pr_message(&winner, None, true)
21402 .await
21403 .unwrap()
21404 .expect("rewritten");
21405 assert_eq!(runner.state.seats["impl-A"].turns, 2);
21406 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
21407 }
21408
21409 #[tokio::test]
21410 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
21411 crate::run::pin_test_home();
21414 let dir = tempfile::tempdir().unwrap();
21415 let mut exhausted = spec("alpha");
21416 exhausted.command = vec!["false".into()];
21417 let mut runner = rewrite_runner(
21418 dir.path(),
21419 false,
21420 vec![exhausted, scripted("beta", EN_REWRITE)],
21421 );
21422 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21424 "alpha".into(),
21425 "beta".into(),
21426 ]));
21427 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21428 seat.turns = 1;
21429 runner.state.seats.insert("impl-A".into(), seat);
21430 let winner = runner.state.candidates[0].clone();
21431 let msg = runner
21432 .guarded_pr_message(&winner, None, true)
21433 .await
21434 .unwrap()
21435 .expect("the role rewrites after the implementer fails");
21436 assert!(msg.body.contains("`src/net.rs`"));
21437 assert_eq!(
21438 runner.state.seats["impl-A"].turns, 2,
21439 "the implementer was tried once"
21440 );
21441 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21442 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21443 }
21444
21445 #[tokio::test]
21446 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
21447 crate::run::pin_test_home();
21448 let dir = tempfile::tempdir().unwrap();
21449 let mut broken = spec("bad");
21450 broken.command = vec!["false".into()];
21451 let mut runner = rewrite_runner(
21452 dir.path(),
21453 true,
21454 vec![
21455 broken,
21456 scripted("noformat", "Add retries."),
21457 scripted("good", EN_REWRITE),
21458 ],
21459 );
21460 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21461 "bad".into(),
21462 "noformat".into(),
21463 "good".into(),
21464 ]));
21465 let winner = runner.state.candidates[0].clone();
21466 let msg = runner
21467 .guarded_pr_message(&winner, None, true)
21468 .await
21469 .unwrap()
21470 .expect("the third agent rewrites");
21471 assert!(msg.body.contains("`src/net.rs`"));
21472 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
21473 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
21474 }
21475
21476 #[tokio::test]
21477 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
21478 crate::run::pin_test_home();
21479 for (agents, role, saved) in [
21481 (
21482 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
21483 Some("beta"),
21484 true,
21485 ),
21486 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
21487 (Vec::new(), None, false),
21488 ] {
21489 let dir = tempfile::tempdir().unwrap();
21490 let mut runner = rewrite_runner(dir.path(), true, agents);
21491 runner.state.config.roles.language_judge =
21492 role.map(|r| crate::config::AgentChoice::One(r.into()));
21493 let winner = runner.state.candidates[0].clone();
21494 let first = runner
21495 .guarded_pr_message(&winner, None, true)
21496 .await
21497 .unwrap();
21498 assert!(first.is_none(), "withheld text parks the run");
21499 assert!(runner.state.parked);
21500 assert_eq!(gate_questions(&runner), 1);
21501 let g = runner.state.github_text.as_ref().unwrap();
21502 assert_eq!(g.rewrite.is_some(), saved);
21503 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
21504 }
21505 }
21506
21507 #[tokio::test]
21508 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
21509 crate::run::pin_test_home();
21510 let dir = tempfile::tempdir().unwrap();
21511 let mut runner = runner_at(RunStatus::Gating);
21512 runner.state =
21513 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
21514 runner.state.repo = dir.path().to_owned();
21515 runner.state.candidates[0].worktree = dir.path().to_owned();
21516 let mut author = spec("alpha");
21517 author.command = vec![
21518 "sh".into(),
21519 "-c".into(),
21520 "printf '%s' \"$REWRITE_REPLY\"".into(),
21521 ];
21522 author.env.insert(
21523 "REWRITE_REPLY".into(),
21524 format!(
21525 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
21526 "English sentence. ".repeat(600)
21527 ),
21528 );
21529 runner.state.config.agents = vec![author];
21530 seed_public_summary(&mut runner);
21531 let winner = runner.state.candidates[0].clone();
21532 assert!(
21533 runner
21534 .guarded_pr_message(&winner, None, true)
21535 .await
21536 .unwrap()
21537 .is_none()
21538 );
21539 let record = runner.state.github_text.clone().unwrap();
21540 let rewrite = record.rewrite.clone().unwrap();
21541 assert_eq!(rewrite.failed, ["body-language"]);
21542 assert!(
21543 rewrite
21544 .body
21545 .ends_with(&format!("magi:run/{}", runner.state.id))
21546 );
21547 let store = ask::Questions::open();
21548 let q = store.get(record.question.as_ref().unwrap()).unwrap();
21549 assert!(
21550 q.choices
21551 .contains(&crate::github_text::USE_REWRITE.to_owned())
21552 );
21553 for label in [
21554 "ORIGINAL",
21555 "FALLBACK",
21556 "REWRITE",
21557 "Rules still failed: body-language",
21558 "truncated",
21559 ] {
21560 assert!(q.detail.contains(label), "{label}");
21561 }
21562 let artifact = std::fs::read_to_string(crate::run::artifact_path(
21563 &runner.state,
21564 &crate::github_text::artifact_name(&record.fingerprint),
21565 ))
21566 .unwrap();
21567 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
21568 runner.state =
21570 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
21571 store
21572 .update(&q.id, |q| {
21573 q.say("まだ日本語")?;
21574 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
21575 })
21576 .unwrap();
21577 assert!(
21578 runner
21579 .guarded_pr_message(&winner, None, true)
21580 .await
21581 .unwrap()
21582 .is_none()
21583 );
21584 let retry = runner.state.github_text.as_ref().unwrap();
21585 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
21586 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
21587 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
21588 assert!(
21589 q.posting_comparison
21590 .as_ref()
21591 .unwrap()
21592 .matches_detail(&q.detail)
21593 );
21594 store
21595 .update(&q.id, |q| {
21596 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21597 })
21598 .unwrap();
21599 let message = runner
21600 .guarded_pr_message(&winner, None, true)
21601 .await
21602 .unwrap()
21603 .unwrap();
21604 assert_eq!(message.title, rewrite.title);
21605 assert_eq!(message.body, rewrite.body);
21606 assert_eq!(
21607 runner.state.github_text.as_ref().unwrap().choice,
21608 Some(GithubTextChoice::Rewrite)
21609 );
21610 let again = runner
21611 .guarded_pr_message(&winner, None, true)
21612 .await
21613 .unwrap()
21614 .unwrap();
21615 assert_eq!(again.title, message.title);
21616 assert_eq!(again.body, message.body);
21617 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21618 assert!(
21619 !runner
21620 .state
21621 .events
21622 .iter()
21623 .any(|e| e.message.contains(&rewrite.title)
21624 || e.message.contains("English sentence."))
21625 );
21626 }
21627
21628 #[tokio::test]
21629 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
21630 crate::run::pin_test_home();
21631 let dir = tempfile::tempdir().unwrap();
21632 let mut runner = runner_at(RunStatus::Gating);
21633 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
21634 runner.state.repo = dir.path().to_owned();
21635 runner.state.candidates[0].worktree = dir.path().to_owned();
21636 seed_public_summary(&mut runner);
21637 let winner = runner.state.candidates[0].clone();
21638 assert!(
21639 runner
21640 .guarded_pr_message(&winner, None, true)
21641 .await
21642 .unwrap()
21643 .is_none()
21644 );
21645 let record = runner.state.github_text.as_ref().unwrap();
21646 assert!(record.rewrite.is_none());
21647 let mut old = serde_json::to_value(record).unwrap();
21649 old.as_object_mut().unwrap().remove("rewrite");
21650 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
21651 assert!(old.rewrite.is_none());
21652 let id = record.question.clone().unwrap();
21653 ask::Questions::open()
21654 .update(&id, |q| {
21655 q.choices.push(crate::github_text::USE_REWRITE.into());
21656 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21657 })
21658 .unwrap();
21659 let message = runner
21660 .guarded_pr_message(&winner, None, true)
21661 .await
21662 .unwrap()
21663 .unwrap();
21664 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21665 assert_eq!(
21666 runner.state.github_text.as_ref().unwrap().choice,
21667 Some(GithubTextChoice::Fallback)
21668 );
21669 }
21670
21671 #[tokio::test]
21672 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
21673 crate::run::pin_test_home();
21674 let dir = tempfile::tempdir().unwrap();
21675 let mut runner = runner_at(RunStatus::Gating);
21676 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
21677 runner.state.repo = dir.path().to_owned();
21678 runner.state.candidates[0].worktree = dir.path().to_owned();
21679 seed_public_summary(&mut runner);
21680 let winner = runner.state.candidates[0].clone();
21681 assert!(
21682 runner
21683 .guarded_pr_message(&winner, None, true)
21684 .await
21685 .unwrap()
21686 .is_none()
21687 );
21688 let store = ask::Questions::open();
21689 let find = |run: &str| {
21690 store
21691 .list()
21692 .into_iter()
21693 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
21694 .unwrap()
21695 };
21696 let run = runner.state.id.clone();
21697 let q = find(&run);
21698 let fp = runner
21699 .state
21700 .github_text
21701 .as_ref()
21702 .unwrap()
21703 .fingerprint
21704 .clone();
21705 let name = crate::github_text::artifact_name(&fp);
21706 let path = crate::run::artifact_path(&runner.state, &name);
21707 assert!(q.detail.contains(&path.display().to_string()));
21708 assert!(q.detail.contains("title-language: title"));
21709 let saved = std::fs::read_to_string(&path).expect("artifact written");
21710 assert!(saved.contains("日本語のタイトル"));
21711 assert!(q.cwd.is_none());
21712 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
21713 let say = |id: &str, text: &str, choice: &str| {
21715 store
21716 .update(id, |q| {
21717 q.thread.push(ask::Turn {
21718 who: ask::Who::Operator,
21719 body: text.to_owned(),
21720 at: jiff::Timestamp::now(),
21721 note: None,
21722 });
21723 q.answer(ask::Answer::Choice(choice.to_owned()))
21724 })
21725 .unwrap();
21726 };
21727 say(&q.id, "まだ日本語", "use my text");
21728 assert!(
21729 runner
21730 .guarded_pr_message(&winner, None, true)
21731 .await
21732 .unwrap()
21733 .is_none()
21734 );
21735 let second = find(&run);
21736 assert_ne!(second.id, q.id);
21737 assert!(second.detail.contains("did not pass"));
21738 assert!(second.detail.contains("日本語のタイトル"));
21741 assert!(!second.detail.contains("まだ日本語"));
21742 assert!(second.detail.contains(&path.display().to_string()));
21743 say(&second.id, "fix: retry failed requests", "use my text");
21744 let message = runner
21745 .guarded_pr_message(&winner, None, true)
21746 .await
21747 .unwrap()
21748 .expect("a vetted replacement is posted");
21749 assert_eq!(message.title, "fix: retry failed requests");
21750 assert_eq!(
21751 runner
21752 .state
21753 .github_text
21754 .as_ref()
21755 .unwrap()
21756 .chosen_title
21757 .as_deref(),
21758 Some("fix: retry failed requests")
21759 );
21760 assert!(runner.state.github_text.as_ref().unwrap().resolved);
21761 let again = runner
21763 .guarded_pr_message(&winner, None, true)
21764 .await
21765 .unwrap()
21766 .unwrap();
21767 assert_eq!(again.title, message.title);
21768 assert_eq!(
21769 store
21770 .list()
21771 .into_iter()
21772 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21773 .count(),
21774 2
21775 );
21776 }
21777
21778 #[test]
21779 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21780 let mut state = state_with_summary(
21784 "add retries",
21785 "TITLE: fix(web): batch reads\n- reads run.json once",
21786 );
21787 state.config.graph.language = "ja".to_owned();
21788 let m = pr_message(&state, 'A');
21789 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21790 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21791
21792 let task = "今回やってほしいこと: results projector を直す";
21795 let mut state = state_with_summary(task, "- no title line");
21796 state.config.graph.language = "ja".to_owned();
21797 let m = pr_message(&state, 'A');
21798 assert_eq!(
21799 m.title,
21800 format!("chore: land candidate A of run {}", state.id)
21801 );
21802 assert!(
21803 m.body.contains(&format!(
21804 "<summary>Original task</summary>\n\n{task}\n\n</details>"
21805 )),
21806 "{}",
21807 m.body
21808 );
21809 }
21810
21811 #[test]
21812 fn pr_message_scrubs_home_paths_and_addresses() {
21813 let state = state_with_summary(
21814 "fix it in /Users/someone/src/x",
21815 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21816 );
21817 let m = pr_message(&state, 'A');
21818 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21819 assert!(!m.body.contains(leak), "{}", m.body);
21820 }
21821 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21822 }
21823
21824 #[test]
21825 fn pr_message_survives_a_task_that_closes_details() {
21826 let state = state_with_summary("a </details> b", "TITLE: fix: x");
21827 let m = pr_message(&state, 'A');
21828 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21829 }
21830
21831 #[test]
21832 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21833 let cmd = manual_merge_command(
21834 MergeStyle::Squash,
21835 Path::new("/repo"),
21836 "b",
21837 "fix: \"quoted\" $(x) `y`\n\nbody",
21838 );
21839 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21840 }
21841
21842 #[test]
21843 fn manual_merge_command_matches_the_configured_style() {
21844 let repo = Path::new("/repo");
21845 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21846
21847 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21848 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21849
21850 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21851 assert_eq!(
21852 squash,
21853 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21854 \"Merge magi run 0832 (candidate A)\""
21855 );
21856
21857 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21858 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21859 }
21860
21861 #[test]
21862 fn a_nudge_gets_a_quarter_of_the_budget() {
21863 assert_eq!(retry_budget(secs(1200), true), secs(300));
21865 assert_eq!(retry_budget(secs(3600), true), secs(900));
21866 }
21867
21868 #[test]
21869 fn a_resent_prompt_keeps_the_whole_budget() {
21870 assert_eq!(retry_budget(secs(1200), false), secs(1200));
21873 assert_eq!(retry_budget(secs(60), false), secs(60));
21874 }
21875
21876 #[test]
21877 fn the_floor_never_exceeds_the_original_budget() {
21878 assert_eq!(retry_budget(secs(60), true), secs(60));
21882 assert_eq!(retry_budget(secs(480), true), secs(120));
21883 assert_eq!(retry_budget(secs(0), true), secs(0));
21884 }
21885
21886 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21887 agent::CommandEvidence {
21888 id: "item1".to_owned(),
21889 description: "cargo test".to_owned(),
21890 exit_code,
21891 result_summary: String::new(),
21892 source: "codex".to_owned(),
21893 }
21894 }
21895
21896 #[test]
21897 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21898 assert!(!has_unconfirmed_command(&[]));
21902 }
21903
21904 #[test]
21905 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21906 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21910 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21911 assert!(!has_unconfirmed_command(&[
21912 evidence(Some(0)),
21913 evidence(Some(101))
21914 ]));
21915 }
21916
21917 #[test]
21918 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21919 assert!(has_unconfirmed_command(&[
21920 evidence(Some(0)),
21921 evidence(None)
21922 ]));
21923 }
21924
21925 #[test]
21926 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21927 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21928 assert_eq!(
21929 verified_noop_claim(true, &[], text).as_deref(),
21930 Some("already fixed by b32cfc4, on main.")
21931 );
21932 }
21933
21934 #[test]
21935 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21936 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21939 assert!(verified_noop_claim(false, &[], text).is_none());
21940 }
21941
21942 #[test]
21943 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21944 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21945 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21946 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21948 }
21949
21950 #[test]
21951 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21952 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21953 }
21954
21955 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21958 runner.state.candidates = shape
21959 .iter()
21960 .enumerate()
21961 .map(|(i, &(empty, verified))| Candidate {
21962 index: i,
21963 label: (b'A' + i as u8) as char,
21964 agent: "sonnet".to_owned(),
21965 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21966 worktree: PathBuf::from(format!("/wt/{i}")),
21967 summary: String::new(),
21968 stat: String::new(),
21969 files: 0,
21970 commits: 0,
21971 empty,
21972 failed: None,
21973 verified_noop: verified.map(str::to_owned),
21974 duration_ms: 0,
21975 folded: false,
21976 })
21977 .collect();
21978 }
21979
21980 #[test]
21981 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21982 ask_test_home();
21983 let mut runner = runner_at(RunStatus::Implementing);
21984 set_candidates(
21985 &mut runner,
21986 &[
21987 (true, Some("already on main at b32cfc4")),
21988 (true, Some("same fix, see the existing test")),
21989 ],
21990 );
21991
21992 runner
21993 .after_implement()
21994 .expect("a verified no-op is not an error");
21995
21996 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21997 }
21998
21999 #[test]
22000 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
22001 ask_test_home();
22002 let mut runner = runner_at(RunStatus::Implementing);
22003 set_candidates(
22007 &mut runner,
22008 &[(true, Some("already on main at b32cfc4")), (true, None)],
22009 );
22010
22011 let err = runner
22012 .after_implement()
22013 .expect_err("an unverified empty candidate must still fail the run");
22014
22015 assert!(
22016 err.to_string().contains("no candidate produced a change"),
22017 "{err}"
22018 );
22019 assert_eq!(runner.state.status, RunStatus::Failed);
22020 }
22021
22022 #[test]
22023 fn after_implement_still_fails_an_ordinary_all_empty_run() {
22024 ask_test_home();
22025 let mut runner = runner_at(RunStatus::Implementing);
22026 set_candidates(&mut runner, &[(true, None), (true, None)]);
22027
22028 let err = runner
22029 .after_implement()
22030 .expect_err("no candidate declared anything; this is an ordinary failure");
22031
22032 assert!(
22033 err.to_string().contains("no candidate produced a change"),
22034 "{err}"
22035 );
22036 assert_eq!(runner.state.status, RunStatus::Failed);
22037 }
22038}
22039
22040#[derive(Debug)]
22043struct GraftRestoreFailed(String);
22044
22045impl std::fmt::Display for GraftRestoreFailed {
22046 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22047 write!(
22048 f,
22049 "could not restore the winner to its pre-graft commit: {}",
22050 self.0
22051 )
22052 }
22053}
22054
22055impl std::error::Error for GraftRestoreFailed {}
22056
22057#[cfg(test)]
22061mod resource_wait_tests {
22062 use super::*;
22063 use std::collections::BTreeMap;
22064
22065 fn sh(id: &str, script: &str) -> AgentSpec {
22066 AgentSpec {
22067 id: id.to_owned(),
22068 kind: crate::config::AgentKind::Command,
22069 model: None,
22070 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
22071 extra_args: Vec::new(),
22072 env: BTreeMap::new(),
22073 prompt_delivery: None,
22074 max_turn_secs: None,
22075 }
22076 }
22077
22078 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
22079 SeatJob {
22080 seat: SeatState::new("impl-A", &spec.id, 1),
22081 spec,
22082 cwd: dir.to_path_buf(),
22083 prompt: "do it".to_owned(),
22084 timeout,
22085 allow_write: true,
22086 sessions: false,
22087 artifacts: dir.join("art"),
22088 stem: "impl-A".to_owned(),
22089 handover: None,
22090 }
22091 }
22092
22093 fn state(resource_wait_secs: u64) -> RunState {
22094 let mut config = Config::default();
22095 config.graph.timeout_resource_wait = resource_wait_secs;
22096 RunState::new(
22097 PathBuf::from("/repo"),
22098 "main".to_owned(),
22099 "abc1234".to_owned(),
22100 "task".to_owned(),
22101 config,
22102 )
22103 }
22104
22105 fn hold(cache: &Path) -> crate::cache::Guard {
22106 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
22107 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
22108 crate::cache::AcquireOutcome::Acquired(g) => g,
22109 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
22110 }
22111 }
22112
22113 async fn run(
22114 jobs: Vec<SeatJob>,
22115 sem: Arc<Semaphore>,
22116 cache: Option<&Path>,
22117 st: &mut RunState,
22118 ) -> Vec<(usize, SeatState, AgentOutcome)> {
22119 let prompts = Prompts::default();
22120 let ctx = WaveCtx {
22121 run: "r",
22122 node: "implement",
22123 prompts: &prompts,
22124 cache,
22125 round: None,
22126 carry_seats: false,
22127 pauses: None,
22128 };
22129 wave(jobs, sem, &ctx, st, 0).await
22130 }
22131
22132 #[tokio::test]
22133 async fn verification_cache_wait_preserves_the_command_budget() {
22134 crate::run::pin_test_home();
22135 let tmp = tempfile::tempdir().unwrap();
22136 let cache = tmp.path().join("target");
22137 let guard = hold(&cache);
22138 let release = tokio::spawn(async move {
22139 tokio::time::sleep(Duration::from_millis(100)).await;
22140 guard.release();
22141 });
22142 let mut st = state(10);
22143 let budget = Duration::from_millis(50);
22144 let wait = repository_wait(&st, None);
22145 let (outcomes, _) = with_cache_lease(
22146 &mut st,
22147 Some(&cache),
22148 "gate",
22149 "gate",
22150 tmp.path(),
22151 "abc1234",
22152 budget,
22153 "test gate",
22154 &wait,
22155 |_, available| async move {
22156 assert_eq!(
22157 available, budget,
22158 "contention must not shorten verification"
22159 );
22160 (Vec::new(), false, Vec::new())
22161 },
22162 )
22163 .await
22164 .unwrap();
22165 release.await.unwrap();
22166 assert!(outcomes.is_empty(), "{outcomes:?}");
22167 assert!(
22168 st.events
22169 .iter()
22170 .any(|event| event.message.contains("waiting for"))
22171 );
22172 assert!(st.resource_wait.is_none());
22173 hold(&cache).release();
22175 }
22176
22177 #[tokio::test]
22178 async fn verification_cache_wait_expires_without_starting_commands() {
22179 crate::run::pin_test_home();
22180 let tmp = tempfile::tempdir().unwrap();
22181 let cache = tmp.path().join("target");
22182 let _held = hold(&cache);
22183 let mut st = state(0);
22184 let wait = repository_wait(&st, None);
22185 let (outcomes, _) = with_cache_lease(
22186 &mut st,
22187 Some(&cache),
22188 "gate",
22189 "gate",
22190 tmp.path(),
22191 "abc1234",
22192 Duration::from_secs(60),
22193 "test gate",
22194 &wait,
22195 |_, _| async { panic!("commands must not start without the cache lease") },
22196 )
22197 .await
22198 .unwrap();
22199 assert_eq!(outcomes.len(), 1);
22200 assert!(outcomes[0].resource_blocked);
22201 assert!(st.resource_wait.is_none());
22202 assert!(matches!(
22203 st.resource_block.unwrap().cause,
22204 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
22205 ));
22206 }
22207
22208 #[tokio::test]
22209 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
22210 crate::run::pin_test_home();
22211 let tmp = tempfile::tempdir().unwrap();
22212 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
22213 let cache = tmp.path().join("cache");
22214 let mut state = state(21600);
22215 let wait = repository_wait(&state, None);
22216 let (outcomes, _) = with_cache_lease(
22217 &mut state,
22218 Some(&cache),
22219 "gate",
22220 "gate",
22221 tmp.path(),
22222 "head",
22223 Duration::from_secs(60),
22224 "gate",
22225 &wait,
22226 |_, _| async { panic!("freshness failure must not start verification") },
22227 )
22228 .await
22229 .unwrap();
22230 assert_eq!(outcomes.len(), 1);
22231 assert!(outcomes[0].resource_blocked);
22232 let block = state.resource_block.unwrap();
22233 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
22234 assert!(!block.description().contains("timed out"));
22235 }
22236
22237 #[tokio::test]
22238 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
22239 crate::run::pin_test_home();
22240 let tmp = tempfile::tempdir().unwrap();
22241 let cache = tmp.path().join("cache");
22242 let _held = hold(&cache);
22243 let mut state = state(21600);
22244 let pause = Pause::new();
22245 let interrupt = Pause::new();
22246 let cause = ParkCause::Operator;
22247 pause.park_for(cause.clone());
22248 let prompts = Prompts::default();
22249 let ctx = WaveCtx {
22250 run: "test",
22251 node: "fix",
22252 prompts: &prompts,
22253 cache: Some(&cache),
22254 round: None,
22255 carry_seats: false,
22256 pauses: Some((&pause, &interrupt)),
22257 };
22258 let result = wave(
22259 vec![job(
22260 sh("writer", "exit 99"),
22261 tmp.path(),
22262 Duration::from_secs(1),
22263 )],
22264 Arc::new(Semaphore::new(1)),
22265 &ctx,
22266 &mut state,
22267 0,
22268 )
22269 .await;
22270 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
22271 assert_eq!(result[0].1.turns, 0);
22272 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22273 assert_eq!(
22274 state.resource_block.unwrap().cause,
22275 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
22276 );
22277 assert!(state.active.is_empty());
22278 }
22279
22280 #[tokio::test]
22281 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
22282 crate::run::pin_test_home();
22283 let tmp = tempfile::tempdir().unwrap();
22284 let cache = tmp.path().join("target-a");
22285 let guard = hold(&cache);
22286 tokio::spawn(async move {
22288 tokio::time::sleep(Duration::from_millis(1500)).await;
22289 guard.release();
22290 });
22291 let mut st = state(60);
22292 let results = run(
22293 vec![job(
22294 sh("alpha", "echo ok"),
22295 tmp.path(),
22296 Duration::from_secs(1),
22297 )],
22298 Arc::new(Semaphore::new(1)),
22299 Some(&cache),
22300 &mut st,
22301 )
22302 .await;
22303 assert!(
22304 matches!(results[0].2, AgentOutcome::Ok(_)),
22305 "the wait must not be charged to the agent"
22306 );
22307 assert!(st.resource_block.is_none());
22308 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22309 assert!(st.resource_wait.is_none() && st.active.is_empty());
22310 }
22311
22312 #[tokio::test]
22313 #[cfg(unix)]
22314 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
22315 use std::os::fd::AsRawFd as _;
22316 crate::run::pin_test_home();
22317 let tmp = tempfile::tempdir().unwrap();
22318 crate::git::git(tmp.path(), &["init"]).await.unwrap();
22319 let common = tmp.path().join(".git");
22320 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
22321 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
22322 assert_eq!(
22323 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
22324 0
22325 );
22326 let mut st = state(1);
22327 st.config.coordination = crate::config::test_coordination();
22328 let mut reader = job(
22329 sh("reader", "echo must-not-run"),
22330 tmp.path(),
22331 Duration::from_secs(30),
22332 );
22333 reader.allow_write = false;
22334 let results = run(
22335 vec![reader.clone()],
22336 Arc::new(Semaphore::new(1)),
22337 None,
22338 &mut st,
22339 )
22340 .await;
22341 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22342 assert_eq!(results[0].1.turns, 0);
22343 assert!(FailClass::of(&results[0].2).is_none());
22344 assert_eq!(
22345 st.resource_block.as_ref().unwrap().kind,
22346 "repository coordination"
22347 );
22348 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22349 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22350 drop(external);
22351 let editor = crate::repo_lock::SeatLease::for_checkout(
22352 tmp.path(),
22353 true,
22354 crate::config::test_coordination().as_ref(),
22355 )
22356 .await
22357 .unwrap();
22358 let pause = Pause::new();
22359 let interrupt = Pause::new();
22360 let prompts = Prompts::default();
22361 let ctx = WaveCtx {
22362 run: "r",
22363 node: "review",
22364 prompts: &prompts,
22365 cache: None,
22366 round: None,
22367 carry_seats: false,
22368 pauses: Some((&pause, &interrupt)),
22369 };
22370 let (results, ()) = tokio::join!(
22371 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
22372 async {
22373 tokio::time::sleep(Duration::from_millis(50)).await;
22374 pause.park();
22375 }
22376 );
22377 assert!(
22378 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
22379 );
22380 assert_eq!(results[0].1.turns, 0);
22381 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22382 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22383 drop(editor);
22384 }
22385
22386 #[tokio::test]
22387 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
22388 crate::run::pin_test_home();
22389 let tmp = tempfile::tempdir().unwrap();
22390 let cache = tmp.path().join("target-b");
22391 let _held = hold(&cache);
22392 let mut st = state(1);
22393 let results = run(
22394 vec![job(
22395 sh("alpha", "echo ok"),
22396 tmp.path(),
22397 Duration::from_secs(1),
22398 )],
22399 Arc::new(Semaphore::new(1)),
22400 Some(&cache),
22401 &mut st,
22402 )
22403 .await;
22404 let (_, seat, out) = &results[0];
22405 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
22406 assert!(wave_blocked(&results));
22407 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
22408 assert_eq!(seat.turns, 0, "nothing was asked");
22409 assert!(
22410 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
22411 "no agent was spawned"
22412 );
22413 assert!(FailClass::of(out).is_none(), "not a failure class");
22414 let block = st.resource_block.as_ref().expect("recorded for the resume");
22415 assert_eq!(block.kind, "shared build cache");
22416 assert!(matches!(
22417 block.cause,
22418 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
22419 ));
22420 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22421 assert!(st.active.is_empty());
22422 }
22423
22424 #[tokio::test]
22425 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
22426 crate::run::pin_test_home();
22427 let tmp = tempfile::tempdir().unwrap();
22428 let mut st = state(1);
22429 let results = run(
22430 vec![job(
22431 sh("alpha", "echo ok"),
22432 tmp.path(),
22433 Duration::from_secs(1),
22434 )],
22435 Arc::new(Semaphore::new(0)),
22436 None,
22437 &mut st,
22438 )
22439 .await;
22440 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22441 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
22442 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
22443 }
22444
22445 #[tokio::test]
22446 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
22447 crate::run::pin_test_home();
22448 let tmp = tempfile::tempdir().unwrap();
22449 let mut st = state(60);
22450 let results = run(
22451 vec![job(
22452 sh("alpha", "sleep 30"),
22453 tmp.path(),
22454 Duration::from_secs(1),
22455 )],
22456 Arc::new(Semaphore::new(1)),
22457 None,
22458 &mut st,
22459 )
22460 .await;
22461 let out = &results[0].2;
22462 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
22463 let class = FailClass::of(out).expect("a failure");
22464 assert_eq!(class, FailClass::Timeout);
22465 assert!(should_hand_over(None, &class), "existing handover policy");
22466 assert!(st.resource_block.is_none());
22467 }
22468}