1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 if state.startup.as_ref().is_some_and(|s| !s.complete)
1470 || (state.status == RunStatus::Prep
1471 && state.instruction.trim().is_empty()
1472 && state.candidates.is_empty()
1473 && state.prep_complete == Some(false))
1474 {
1475 bail!(
1476 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477 );
1478 }
1479 crate::pr_head::ensure_owner(&state)?;
1480 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481 bail!(
1482 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483 );
1484 }
1485 if let Some(to) = &state.released_to {
1486 bail!(
1487 "run {} cannot be resumed: its worktree was released to run {}",
1488 state.short(),
1489 crate::run::short_of(to)
1490 );
1491 }
1492 let roles = state.config.resolve_roles()?;
1493 if let Some(id) = &state.config.graph.selected_implementer
1494 && state.candidates.iter().any(|c| c.agent != *id)
1495 {
1496 bail!(
1497 "replacement implementer pin does not match the saved candidates; task must stay held"
1498 );
1499 }
1500 let max_parallel = state.config.graph.max_parallel.max(1);
1501 Ok(Self {
1502 state,
1503 roles,
1504 sem: Arc::new(Semaphore::new(max_parallel)),
1505 pause: Pause::new(),
1506 interrupt: Pause::new(),
1507 })
1508 }
1509
1510 pub async fn execute(&mut self) -> Result<()> {
1517 anyhow::ensure!(
1518 self.state.startup.as_ref().is_none_or(|s| s.complete),
1519 "startup incomplete; inspect retained evidence before explicit recovery"
1520 );
1521 if let Some(selection) = &self.state.pr_recovery
1522 && (selection.source_proof.is_some() || selection.source_run.is_some())
1523 {
1524 Box::pin(selection.verify_source(&self.state.repo)).await?;
1525 }
1526 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529 || self.state.land_approval.is_some()
1530 || self.state.merge_decision.is_some()
1531 || !self.state.merge_repairs.is_empty()
1532 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533 {
1534 let claim = FixClaim::acquire(&self.state.dir())?;
1535 let disk = RunState::load(&self.state.id)?;
1536 let checkpoint = |s: &RunState| {
1537 (
1538 s.merge_repairs.len(),
1539 s.merge_repairs
1540 .last()
1541 .map(|r| (r.question.clone(), r.phase)),
1542 s.land_approval
1543 .as_ref()
1544 .map(|a| (a.question.clone(), a.head.clone())),
1545 )
1546 };
1547 if checkpoint(&disk) != checkpoint(&self.state) {
1548 bail!(
1549 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550 );
1551 }
1552 Some(claim)
1553 } else {
1554 None
1555 };
1556 let mut execution = self.claim_driver().await?;
1557 let runner = &mut *execution.runner;
1558 let wait = runner.repository_wait();
1559 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562 let result = match result {
1563 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564 record_repository_block(&mut runner.state, &e);
1565 runner.park_here()?;
1566 Ok(())
1567 }
1568 other => other,
1569 };
1570 execution.driver.finish(&mut runner.state).await?;
1571 let ended = if result.is_err() {
1572 Some({
1573 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574 if let Err(error) = &result {
1575 notice.message = format!(
1576 "Run {} stopped with an error: {}",
1577 runner.state.id,
1578 error.to_string().lines().next().unwrap_or("unknown error")
1579 );
1580 }
1581 notice
1582 })
1583 } else {
1584 crate::notices::run_ended(&runner.state)
1585 };
1586 if let Some(notice) = ended {
1587 crate::notices::raise(notice);
1588 }
1589 result
1590 }
1591
1592 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594 }
1595
1596 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597 let wait = self.repository_wait();
1598 coordinate_operation(&mut self.state, cwd, &wait).await
1599 }
1600
1601 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603 let owner = match crate::pr_head::owner_lock_wait(
1604 &crate::run::home(),
1605 &self.state.id,
1606 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607 )
1608 .await
1609 {
1610 Ok(owner) => owner,
1611 Err(error) => {
1612 let detail = format!("worktree maintenance or ownership update: {error:#}");
1613 self.state.resource_block = Some(crate::run::ResourceBlock {
1617 cause: crate::run::ResourceBlockCause::Other,
1618 node: "start".into(),
1619 kind: "worktree maintenance".into(),
1620 detail: detail.clone(),
1621 at: Timestamp::now(),
1622 });
1623 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624 }
1625 };
1626 let disk = RunState::load(&self.state.id)?;
1627 crate::pr_head::ensure_owner(&disk)?;
1628 let claimed =
1629 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630 anyhow::ensure!(
1635 disk.liveness(claimed) != crate::run::Liveness::Live,
1636 "run already has a live driver; wait for its execution to end"
1637 );
1638 if let Some(to) = &disk.released_to {
1639 bail!(
1640 "run {} cannot continue: its worktree was released to run {}",
1641 self.state.short(),
1642 crate::run::short_of(to)
1643 );
1644 }
1645 anyhow::ensure!(
1646 serde_json::to_value(&disk.base_sync_recovery)?
1647 == serde_json::to_value(&self.state.base_sync_recovery)?,
1648 "base recovery changed since this run was loaded; reload the saved run"
1649 );
1650 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651 let mut execution = DriverExecution {
1652 runner: self,
1653 driver,
1654 };
1655 let _owner = owner;
1658 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659 .await?;
1660 let state = &mut execution.runner.state;
1661 state.parked = false;
1666 state.resource_wait = None;
1667 state.resource_block = None;
1668 state.clear_active();
1675 let pid = std::process::id();
1690 state.driver_pid = Some(pid);
1691 state.driver_started_at = crate::proc::process_started_at(pid);
1692 state.driver_exited = false;
1693 state.driver_claimed = Some(true);
1694 state.begin_pass();
1695 execution.driver.bind(state);
1696 state.save()?;
1697 execution.driver.record(state)?;
1698 Ok(execution)
1699 }
1700
1701 async fn execute_granted_repair(&mut self) -> Result<()> {
1704 use crate::repair_budget::{Kind, Phase};
1705 let index = self.state.repair_grants.len() - 1;
1706 let winner = self
1707 .state
1708 .winner()
1709 .cloned()
1710 .context("repair winner missing")?;
1711 loop {
1712 let grant = self.state.repair_grants[index].clone();
1713 match grant.phase {
1714 Phase::Pending => {
1715 if grant.spent >= grant.receipt.request.rounds {
1716 self.state.repair_grants[index].phase = Phase::Complete;
1717 self.state.status = RunStatus::Blocked;
1718 self.state.save()?;
1719 return Ok(());
1720 }
1721 self.state.repair_grants[index].spent += 1;
1724 self.state.repair_grants[index].phase = Phase::Fixing;
1725 self.state.save()?;
1726 match grant.receipt.request.kind {
1727 Kind::Gate => {
1728 let failed = self.state.gate.clone();
1729 let _ = self.gate_fix_round(&winner, &failed).await?;
1730 }
1731 Kind::Review => {
1732 let mut record = self
1733 .state
1734 .reviews
1735 .last()
1736 .cloned()
1737 .context("review failure missing")?;
1738 let round = record.round;
1739 let step = self.run_fix_round(&winner, &mut record).await?;
1740 self.state.reviews[round - 1] = record;
1741 if matches!(step, FixStep::Recorded { report_lost: true }) {
1742 self.state.repair_grants[index].phase = Phase::Complete;
1743 self.state.status = RunStatus::Blocked;
1744 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745 self.state.save()?;
1746 return Ok(());
1747 }
1748 if matches!(step, FixStep::Resource) {
1749 self.stop_for_resource()?;
1750 return Ok(());
1751 }
1752 self.state.save()?;
1753 }
1754 }
1755 if self.resource_halt()? {
1756 return Ok(());
1757 }
1758 }
1759 Phase::Fixing => {
1760 let _coordination = self.coordinate(&winner.worktree).await?;
1763 let rescued = git::rescue_commit(
1764 &winner.worktree,
1765 "magi: preserve interrupted granted repair",
1766 )
1767 .await?;
1768 self.state.note_withheld("repair-grant", &rescued.withheld);
1769 self.state
1772 .verification_history
1773 .push(crate::run::VerificationEvidence {
1774 superseded_by: None,
1775 archived_at: Some(Timestamp::now()),
1776 fixes_spent: Some(self.state.gate_fixes.len()),
1777 gate_head: self.state.gate_head.take(),
1778 gate_ran: std::mem::take(&mut self.state.gate_ran),
1779 gate: std::mem::take(&mut self.state.gate),
1780 e2e: self.state.head_e2e.take(),
1781 });
1782 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783 self.state
1784 .verification_history
1785 .push(crate::run::VerificationEvidence {
1786 superseded_by: None,
1787 archived_at: Some(Timestamp::now()),
1788 fixes_spent: Some(self.state.gate_fixes.len()),
1789 gate_head: None,
1790 gate_ran: false,
1791 gate: Vec::new(),
1792 e2e: Some(e2e),
1793 });
1794 }
1795 self.state.repair_grants[index].review_start = self.state.reviews.len();
1796 self.state.repair_grants[index].head =
1797 git::rev_parse(&winner.worktree, "HEAD").await?;
1798 self.state.repair_grants[index].phase = Phase::Reviewing;
1799 self.state.save()?;
1800 }
1801 Phase::Reviewing => {
1802 self.review_loop_pass().await?;
1803 if self.resource_halt()? {
1804 return Ok(());
1805 }
1806 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807 let clean = self.state.reviews.len() > grant.review_start
1808 && self.state.reviews.last().is_some_and(|r| {
1809 r.head == head
1810 && r.clean
1811 && r.expected > 0
1812 && r.answered == r.expected
1813 && matches!(
1814 r.e2e_status(),
1815 E2eStatus::Passed | E2eStatus::NotConfigured
1816 )
1817 });
1818 if !clean {
1819 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821 == Kind::Review
1822 && grant.spent < grant.receipt.request.rounds
1823 && self.state.reviews.last().is_some_and(|r| {
1824 r.fix.is_none()
1825 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826 }) {
1827 Phase::Pending
1828 } else {
1829 Phase::Complete
1830 };
1831 self.state.status = RunStatus::Blocked;
1832 self.state.event(
1833 "repair-grant",
1834 "fresh review did not clear the repaired head; no publication",
1835 );
1836 self.state.save()?;
1837 if self.state.repair_grants[index].phase == Phase::Pending {
1838 continue;
1839 }
1840 return Ok(());
1841 }
1842 self.state.repair_grants[index].phase = Phase::Gating;
1843 self.state.save()?;
1844 }
1845 Phase::Gating => {
1846 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847 self.state.reconcile_verification_head(&head);
1848 let wait = self.repository_wait();
1849 if self
1850 .state
1851 .reviews
1852 .last()
1853 .is_none_or(|r| r.head != head || !r.clean)
1854 {
1855 self.state.status = RunStatus::Blocked;
1856 self.state.repair_grants[index].phase = Phase::Complete;
1857 self.state.event(
1858 "repair-grant",
1859 "head changed since fresh review; repair held",
1860 );
1861 self.state.save()?;
1862 return Ok(());
1863 }
1864 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865 return Ok(());
1866 }
1867 let outcomes = self.run_gate(&winner).await?;
1868 if verify_inconclusive(&outcomes) {
1869 return Ok(());
1870 }
1871 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872 if after != head {
1873 self.state.repair_grants[index].review_start = self.state.reviews.len();
1874 self.state.repair_grants[index].head = after;
1875 self.state.repair_grants[index].phase = Phase::Reviewing;
1876 self.state.status = RunStatus::Blocked;
1877 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878 self.state.save()?;
1879 return Ok(());
1880 }
1881 let passed = outcomes.iter().all(CommandOutcome::ok);
1882 self.state.repair_grants[index].succeeded = passed;
1883 self.state.gate = outcomes;
1884 self.state.gate_ran = true;
1885 self.state.status = if passed {
1886 RunStatus::Gating
1887 } else {
1888 RunStatus::Blocked
1889 };
1890 self.state.repair_grants[index].phase = if !passed
1891 && grant.receipt.request.kind == Kind::Gate
1892 && grant.spent < grant.receipt.request.rounds
1893 {
1894 Phase::Pending
1895 } else {
1896 Phase::Complete
1897 };
1898 if passed && self.state.config.merge.mode == MergeMode::Pr {
1899 self.state.pr_publication_pending = true;
1901 } else if passed {
1902 self.state.status = RunStatus::Ready;
1903 }
1904 self.state.save()?;
1905 if !passed {
1906 continue;
1907 }
1908 if self.state.config.merge.mode != MergeMode::Pr {
1912 self.state.status = RunStatus::Ready;
1913 self.state.event(
1914 "repair-grant",
1915 "repair verified locally; grant carries no merge authorization",
1916 );
1917 self.state.save()?;
1918 return Ok(());
1919 }
1920 return self.merge().await;
1921 }
1922 Phase::Complete => return Ok(()),
1923 }
1924 }
1925 }
1926
1927 async fn execute_graph(&mut self) -> Result<()> {
1928 self.state.configure_recovery_resume()?;
1929 if self
1930 .state
1931 .repair_grants
1932 .last()
1933 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1934 {
1935 return self.execute_granted_repair().await;
1936 }
1937 if self
1938 .state
1939 .repair_grants
1940 .last()
1941 .is_some_and(|g| !g.succeeded)
1942 {
1943 self.state.status = RunStatus::Blocked;
1944 self.state.save()?;
1945 return Ok(());
1946 }
1947 self.persist_candidates().await?;
1950 if self.state.base_sync_blocked() {
1953 self.sync_to_base().await?;
1954 if !self.state.base_sync_blocked()
1955 && self.state.status != RunStatus::AlreadyInBase
1956 && !self.resource_halt()?
1957 {
1958 self.finish_after_tally().await?;
1959 }
1960 return Ok(());
1961 }
1962 if self
1963 .state
1964 .merge_decision
1965 .as_ref()
1966 .is_some_and(|decision| !decision.closed)
1967 && self.state.status != RunStatus::Landing
1968 {
1969 if crate::merge_decision::resumable_hold(&self.state) {
1970 self.state.merge = Some(MergeOutcome {
1971 mode: MergeMode::Pr,
1972 ok: false,
1973 detail: self
1974 .state
1975 .merge_decision
1976 .as_ref()
1977 .unwrap()
1978 .target
1979 .pr
1980 .clone(),
1981 empty: false,
1982 });
1983 self.run_land().await?;
1984 self.settle_questions();
1985 return Ok(());
1986 }
1987 bail!(
1988 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1989 );
1990 }
1991 if self.state.config.graph.selected_implementer.is_some()
1992 && self.state.status == RunStatus::Blocked
1993 && self.state.tally.is_none()
1994 && self.state.candidate_preservation.is_empty()
1995 && self.state.interrupted_implementations.is_empty()
1996 {
1997 bail!(
1998 "selected implementation is held; explicitly release the owning task to retry it"
1999 );
2000 }
2001
2002 if self.state.merge_repairs.last().is_some_and(|r| {
2003 matches!(
2004 r.phase,
2005 crate::run::MergeRepairPhase::Requested
2006 | crate::run::MergeRepairPhase::Fixing
2007 | crate::run::MergeRepairPhase::Reviewing
2008 )
2009 }) {
2010 if self.repair_merge_review().await? {
2011 self.run_land().await?;
2012 }
2013 self.settle_questions();
2014 return Ok(());
2015 }
2016 if self.state.pr_publication_pending {
2019 self.merge().await?;
2020 if self.resource_halt()? {
2021 return Ok(());
2022 }
2023 self.state.save()?;
2024 self.settle_questions();
2025 return Ok(());
2026 }
2027 if self.state.status == RunStatus::Stalled {
2040 if self.recover_stall().await? {
2041 self.finish_after_tally().await?;
2042 } else {
2043 self.state.save()?;
2045 }
2046 return Ok(());
2047 }
2048 if self.state.status == RunStatus::Landing
2058 || crate::handover::landing_checkpoint(&self.state)
2059 {
2060 self.state.land_sync_pending = true;
2061 self.state.save()?;
2062 self.run_land().await?;
2063 self.settle_questions();
2068 return Ok(());
2069 }
2070 if self.state.github_text.is_some()
2075 && self.state.merge.is_none()
2076 && self.state.status == RunStatus::Gating
2077 {
2078 self.merge().await?;
2079 if self.resource_halt()? {
2080 return Ok(());
2081 }
2082 self.state.save()?;
2083 return Ok(());
2084 }
2085 if let Err(e) = self.prep().await {
2086 if e.is::<crate::repo_lock::ResourceBlocked>() {
2087 return Err(e);
2088 }
2089 self.state.status = RunStatus::Blocked;
2090 self.state
2091 .prep_failure
2092 .get_or_insert_with(|| format!("{e:#}"));
2093 self.state
2094 .event("prep", format!("preparation stopped: {e:#}"));
2095 self.state.save()?;
2096 return Err(e);
2097 }
2098 if self.park_here()? {
2099 return Ok(());
2100 }
2101 self.advise().await?;
2102 if self.park_here()? {
2103 return Ok(());
2104 }
2105 self.implement().await?;
2106 if self.park_here()? {
2107 return Ok(());
2108 }
2109 if self.state.status == RunStatus::VerifiedNoop {
2113 return Ok(());
2114 }
2115 self.judge().await?;
2116 if self.park_here()? {
2117 return Ok(());
2118 }
2119 self.deliberate().await?;
2120 if self.park_here()? {
2121 return Ok(());
2122 }
2123 self.vote().await?;
2124 if self.park_here()? {
2125 return Ok(());
2126 }
2127 self.tally()?;
2128 if self.state.status == RunStatus::Stalled {
2133 self.state.save()?;
2137 return Ok(());
2138 }
2139 self.finish_after_tally().await?;
2140 Ok(())
2141 }
2142
2143 fn stop_for_resource(&mut self) -> Result<()> {
2148 if self.pause.parked() || self.interrupt.parked() {
2151 self.park_here()?;
2152 return Ok(());
2153 }
2154 if self.state.resource_block.is_some() {
2155 self.state.status = RunStatus::Blocked;
2156 }
2157 self.state.save()
2158 }
2159
2160 fn resource_halt(&mut self) -> Result<bool> {
2162 if self.state.resource_block.is_none() {
2163 return Ok(false);
2164 }
2165 self.stop_for_resource()?;
2166 Ok(true)
2167 }
2168
2169 fn park_here(&mut self) -> Result<bool> {
2176 let parking = self.pause.parked() || self.interrupt.parked();
2177 if !parking && self.state.resource_block.is_some() {
2178 self.stop_for_resource()?;
2179 return Ok(true);
2180 }
2181 if !self.pause.parked() && !self.interrupt.parked() {
2186 return Ok(false);
2187 }
2188 if self.state.parked {
2189 return Ok(true);
2190 }
2191 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2192 if let Some(block) = &mut self.state.resource_block
2193 && matches!(
2194 block.cause,
2195 crate::run::ResourceBlockCause::Cancelled { .. }
2196 )
2197 {
2198 block.cause = crate::run::ResourceBlockCause::Cancelled {
2199 park: cause.clone(),
2200 };
2201 }
2202 let why = match cause.as_ref().map(ParkCause::label) {
2203 Some(reason) => format!(
2204 "parked after `{}` ({reason}) — resume to carry on from here",
2205 self.state.status.as_str()
2206 ),
2207 None => format!(
2208 "parked after `{}` — resume to carry on from here",
2209 self.state.status.as_str()
2210 ),
2211 };
2212 self.state.event("park", why);
2213 self.state
2214 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2215 self.state.parked = true;
2216 self.state.save()?;
2217 Ok(true)
2218 }
2219
2220 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2222 if self.state.prep_failure.is_none() {
2223 return Ok(());
2224 }
2225 let failed_at = self
2226 .state
2227 .events
2228 .iter()
2229 .rev()
2230 .find(|event| event.node == "prep")
2231 .map_or(self.state.updated_at, |event| event.at);
2232 anyhow::ensure!(
2233 approval_at > failed_at,
2234 "preparation retry requires a new explicit release after the failure"
2235 );
2236 self.state.prep_failure = None;
2237 self.state.prep_complete = Some(false);
2238 self.state
2239 .event("prep-retry", "operator approved retry of preparation");
2240 Ok(())
2241 }
2242
2243 pub fn on_pause(&mut self, pause: Pause) {
2245 self.pause = pause;
2246 }
2247
2248 pub fn watch_interrupt(&mut self, pause: Pause) {
2254 self.interrupt = pause;
2255 }
2256
2257 fn settle_questions(&mut self) {
2277 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2278 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2279 }
2280 }
2281
2282 async fn finish_after_tally(&mut self) -> Result<()> {
2285 self.graft().await?;
2288 if self.resource_halt()? {
2289 return Ok(());
2290 }
2291 self.fold_losers().await?;
2292 self.sync_to_base().await?;
2297 if self.hold_base_sync().await? || self.resource_halt()? {
2298 return Ok(());
2299 }
2300 if self.state.status == RunStatus::AlreadyInBase {
2301 return Ok(());
2302 }
2303 self.review_loop().await?;
2304 if self.resource_halt()? {
2305 return Ok(());
2306 }
2307 self.sync_to_base().await?;
2308 if self.hold_base_sync().await? || self.resource_halt()? {
2309 return Ok(());
2310 }
2311 if self.state.status == RunStatus::AlreadyInBase {
2312 return Ok(());
2313 }
2314 self.gate().await?;
2315 self.merge().await?;
2316 if self.resource_halt()? {
2317 return Ok(());
2318 }
2319 self.state.save()?;
2320 Ok(())
2321 }
2322
2323 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2330 let result = match ensure_setup_config(&mut self.state, repo).await {
2331 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2332 Err(e) => Err(e),
2333 };
2334 if let Err(e) = result {
2335 git::worktree_remove_clean(repo, wt).await.ok();
2336 if in_prep {
2337 self.rollback_prep(repo).await;
2338 }
2339 self.state.status = RunStatus::Blocked;
2340 self.state.event(
2341 "setup",
2342 format!("worktree setup failed in {}: {e:#}", wt.display()),
2343 );
2344 self.state.save()?;
2345 return Err(e);
2346 }
2347 Ok(())
2348 }
2349
2350 async fn rollback_prep(&mut self, repo: &Path) {
2352 let root = self.state.worktree_root();
2353 for c in std::mem::take(&mut self.state.candidates) {
2354 if matches!(
2355 git::worktree_remove_clean(repo, &c.worktree).await,
2356 Ok(true)
2357 ) {
2358 git::branch_delete(repo, &c.branch).await.ok();
2359 } else {
2360 self.state.candidates.push(c);
2361 }
2362 }
2363 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2364 for j in 1..=seats {
2365 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2366 .await
2367 .ok();
2368 }
2369 for k in 1..=self.state.config.graph.advisors {
2370 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2371 .await
2372 .ok();
2373 }
2374 }
2375
2376 async fn prep(&mut self) -> Result<()> {
2377 if let Some(reason) = &self.state.prep_failure {
2378 bail!(
2379 "candidate preparation previously failed: {reason}; correct the source before retrying"
2380 );
2381 }
2382 if self
2383 .state
2384 .prep_complete
2385 .unwrap_or(!self.state.candidates.is_empty())
2386 {
2387 return Ok(());
2388 }
2389 let repo = self.state.repo.clone();
2390 let _coordination = self.coordinate(&repo).await?;
2393 let base = self.state.base_commit.clone();
2394 let plan = self.state.sources.plan(&repo, &base).await?;
2395 self.rollback_prep(&repo).await;
2396 if !self.state.candidates.is_empty() {
2397 bail!(
2398 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2399 );
2400 }
2401 self.state.prep_complete = Some(false);
2402 self.state.status = RunStatus::Prep;
2403 self.state.save()?;
2404 let repo = self.state.repo.clone();
2405 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2406 let root = self.state.worktree_root();
2407 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2408
2409 let hooks_dir = self.state.dir().join("hooks");
2412 if self.state.config.blind.commit_msg_hook {
2413 std::fs::create_dir_all(&hooks_dir)
2414 .with_context(|| format!("create {}", hooks_dir.display()))?;
2415 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2416 let path = hooks_dir.join("commit-msg");
2417 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2418 make_executable(&path)?;
2419 git::acquire_worktree_config(&repo).await?;
2427 self.state.enabled_worktree_config = true;
2428 }
2429
2430 ensure_setup_config(&mut self.state, &repo).await?;
2433
2434 for (index, (spec, label)) in self
2435 .roles
2436 .implementers
2437 .clone()
2438 .into_iter()
2439 .zip(labels)
2440 .enumerate()
2441 {
2442 let branch = self.state.branch_for(label);
2443 let worktree = root.join(format!("cand-{label}"));
2444 self.state.event(
2448 "prep-intent",
2449 serde_json::json!({
2450 "branch": branch,
2451 "worktree": worktree,
2452 "start": start,
2453 "picks": plan.picks,
2454 })
2455 .to_string(),
2456 );
2457 self.state.save()?;
2458 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2459 if self.state.config.blind.commit_msg_hook {
2460 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2461 }
2462 git::local_exclude(&worktree, "/.magi/").await?;
2463 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2464 git::worktree_remove_clean(&repo, &worktree).await.ok();
2467 git::branch_delete(&repo, &branch).await.ok();
2468 self.rollback_prep(&repo).await;
2469 self.state.status = RunStatus::Blocked;
2470 self.state
2471 .event("prep", format!("worktree setup failed: {e:#}"));
2472 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2473 self.state.save()?;
2474 return Err(e);
2475 }
2476 self.state.candidates.push(Candidate {
2477 index,
2478 label,
2479 agent: spec.id.clone(),
2480 branch,
2481 worktree,
2482 summary: String::new(),
2483 stat: String::new(),
2484 files: 0,
2485 commits: 0,
2486 empty: false,
2487 failed: None,
2488 verified_noop: None,
2489 duration_ms: 0,
2490 folded: false,
2491 });
2492 self.state.save()?;
2493 for pick in &plan.picks {
2494 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2495 {
2496 self.state.status = RunStatus::Blocked;
2497 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2498 self.state
2499 .event("prep", format!("cannot apply referenced commit: {e}"));
2500 self.state.save()?;
2501 return Err(e);
2502 }
2503 }
2504 }
2505
2506 for j in 1..=self.roles.judges.len() {
2507 let wt = root.join(format!("judge-{j}"));
2508 if !wt.exists() {
2509 git::worktree_add_detached(&repo, &wt, &base).await?;
2510 self.setup_seat_worktree(&repo, &wt, true).await?;
2511 }
2512 }
2513
2514 if self.state.config.graph.advise {
2522 for k in 1..=self.state.config.graph.advisors {
2523 let wt = root.join(format!("advisor-{k}"));
2524 if !wt.exists() {
2525 git::worktree_add_detached(&repo, &wt, &base).await?;
2526 self.setup_seat_worktree(&repo, &wt, true).await?;
2527 }
2528 }
2529 }
2530
2531 let authors: Vec<&str> = self
2536 .roles
2537 .implementers
2538 .iter()
2539 .map(|a| a.id.as_str())
2540 .collect();
2541 let overlap: Vec<String> = self
2542 .roles
2543 .judges
2544 .iter()
2545 .enumerate()
2546 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2547 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2548 .collect();
2549 if !overlap.is_empty() {
2550 let note = format!(
2551 "{} also authored a candidate; blind, but the panel is less \
2552 independent than {} distinct agents would be",
2553 overlap.join(", "),
2554 self.roles.judges.len()
2555 );
2556 self.state.event("prep", note);
2557 }
2558
2559 self.state.event(
2560 "prep",
2561 format!(
2562 "{} candidates, {} judges, base {} ({})",
2563 self.state.candidates.len(),
2564 self.roles.judges.len(),
2565 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2566 self.state.base_branch
2567 ),
2568 );
2569 self.state.prep_complete = Some(true);
2570 self.state.status = RunStatus::Implementing;
2571 self.state.save()?;
2572 Ok(())
2573 }
2574
2575 async fn advise(&mut self) -> Result<()> {
2608 let implement_untouched = self
2609 .state
2610 .candidates
2611 .iter()
2612 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2613 if !self.state.config.graph.advise || self.state.advise_attempted {
2614 return Ok(());
2615 }
2616 if !implement_untouched {
2617 self.state.event(
2618 "advise",
2619 "skipping the design-deliberation stage: at least one \
2620 candidate already shows implementation progress, so this \
2621 run is past the point the stage exists to run before"
2622 .to_owned(),
2623 );
2624 self.state.advise_attempted = true;
2625 self.state.save()?;
2626 return Ok(());
2627 }
2628 let run_id = self.state.id.clone();
2629 let prompts = self.state.config.prompts.clone();
2630 let instruction = self.state.instruction.clone();
2631 let language = self.state.config.graph.language.clone();
2632 let root = self.state.worktree_root();
2633 let n = self.state.config.graph.advisors;
2634 let where_recorded = self.state.dir().join("run.json");
2635
2636 let seats = match self.state.config.advisors() {
2637 Ok(seats) if !seats.is_empty() => seats,
2638 Ok(_) => {
2639 self.state.event(
2640 "advise",
2641 format!(
2642 "[graph] advisors is 0; skipping the design-deliberation \
2643 stage and continuing without a synthesis brief (see {})",
2644 where_recorded.display()
2645 ),
2646 );
2647 self.state.advise_attempted = true;
2648 self.state.save()?;
2649 return Ok(());
2650 }
2651 Err(e) => {
2652 self.state.event(
2653 "advise",
2654 format!(
2655 "could not resolve advisor seats ({e:#}); continuing \
2656 without a design-deliberation brief (see {})",
2657 where_recorded.display()
2658 ),
2659 );
2660 self.state.advise_attempted = true;
2661 self.state.save()?;
2662 return Ok(());
2663 }
2664 };
2665
2666 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2667 let artifacts = agent::artifacts_dir(&self.state.dir());
2668 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2669
2670 let mut jobs = Vec::new();
2671 for (i, spec) in seats.iter().cloned().enumerate() {
2672 let seat_key = format!("advisor-{}", i + 1);
2673 let seat = self.seat(&seat_key, &spec.id);
2674 jobs.push(SeatJob {
2675 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2676 spec,
2677 seat,
2678 cwd: worktrees[i % worktrees.len()].clone(),
2679 timeout,
2680 allow_write: false,
2681 sessions: false,
2682 artifacts: artifacts.clone(),
2683 stem: seat_key,
2684 handover: None,
2685 });
2686 }
2687
2688 self.state.event(
2689 "advise",
2690 format!(
2691 "{} advisor seat(s) sketching a design in parallel",
2692 jobs.len()
2693 ),
2694 );
2695 let mut quota_losses = Vec::new();
2696 let cache = self.state.config.cache_dir();
2697 let ctx = WaveCtx {
2698 carry_seats: false,
2699 pauses: Some((&self.pause, &self.interrupt)),
2700 run: &run_id,
2701 node: "advise",
2702 prompts: &prompts,
2703 cache: cache.as_deref(),
2704 round: None,
2705 };
2706 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2707 let results = ask_json_wave::<Proposal>(
2708 jobs,
2709 Arc::clone(&self.sem),
2710 self.state.config.graph.retries,
2711 &advisor_roster,
2712 &ctx,
2713 &mut quota_losses,
2714 &mut self.state,
2715 &|p: &Proposal| p.validate(),
2716 )
2717 .await;
2718 if self.state.resource_block.is_some() {
2719 return self.stop_for_resource();
2720 }
2721 self.state.quota.extend(quota_losses);
2722
2723 let mut records = Vec::with_capacity(results.len());
2724 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2725 let agent_id = seat.agent.clone();
2726 self.state.seats.insert(seat.key.clone(), seat);
2727 match res {
2728 Ok((proposal, out)) => {
2729 self.state
2730 .event("advise", format!("advisor-{} proposed a design", i + 1));
2731 records.push(advise::AdvisorRecord::proposed(
2732 i + 1,
2733 agent_id,
2734 proposal,
2735 out.duration_ms,
2736 ));
2737 }
2738 Err(e) => {
2739 self.state.event(
2740 "advise",
2741 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2742 );
2743 records.push(advise::AdvisorRecord::failed(
2744 i + 1,
2745 agent_id,
2746 e.to_string(),
2747 ));
2748 }
2749 }
2750 }
2751
2752 let mut advice = advise::Advice {
2753 records,
2754 synthesis: None,
2755 };
2756 if advice.proposals().is_empty() {
2757 self.state.event(
2758 "advise",
2759 "no advisor produced a usable proposal; continuing without a \
2760 synthesis brief"
2761 .to_owned(),
2762 );
2763 } else {
2764 match self
2765 .synthesize_brief(
2766 &advice,
2767 &instruction,
2768 &language,
2769 &worktrees[0],
2770 &artifacts,
2771 &run_id,
2772 &prompts,
2773 cache.as_deref(),
2774 )
2775 .await
2776 {
2777 Ok(Some(text)) => {
2778 self.state.event(
2779 "advise",
2780 "synthesized a design brief for the implementer".to_owned(),
2781 );
2782 advice.synthesis = Some(text);
2783 }
2784 Ok(None) => {
2785 self.state.event(
2786 "advise",
2787 "the synthesis seat produced nothing usable; continuing \
2788 without a design brief"
2789 .to_owned(),
2790 );
2791 }
2792 Err(e) => {
2793 if e.is::<crate::repo_lock::ResourceBlocked>() {
2794 return Err(e);
2795 }
2796 self.state.event(
2797 "advise",
2798 format!("could not synthesize a design brief: {e:#}"),
2799 );
2800 }
2801 }
2802 }
2803 advise::apply_reflection(&mut advice);
2804
2805 self.state.advice = Some(advice);
2806 self.state.advise_attempted = true;
2807 self.state.save()?;
2808 Ok(())
2809 }
2810
2811 #[allow(clippy::too_many_arguments)]
2823 async fn synthesize_brief(
2824 &mut self,
2825 advice: &advise::Advice,
2826 instruction: &str,
2827 language: &str,
2828 cwd: &Path,
2829 artifacts: &Path,
2830 run_id: &str,
2831 prompts: &Prompts,
2832 cache: Option<&Path>,
2833 ) -> Result<Option<String>> {
2834 let chain = agent::pick_chain(
2835 &self.state.config.agents,
2836 self.state.config.roles.synthesizer.as_ref(),
2837 &agent::installed,
2838 "synthesizer",
2839 )?;
2840 let proposals = advice.proposals();
2841 let mut prompt = prompt::with_overlay(
2842 prompt::synthesize_brief(instruction, &proposals, language),
2843 prompts.overlay("advise"),
2844 );
2845 if cache.is_some() {
2846 prompt.push('\n');
2851 prompt.push_str(&prompt::build_cache_note("advise", false));
2852 }
2853 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2854 let mut last = None;
2859 for (n, spec) in chain.iter().enumerate() {
2860 if n > 0 {
2861 self.state
2862 .event("advise", format!("synthesis falling back to {}", spec.id));
2863 }
2864 let mut seat = self.seat("advise-synthesis", &spec.id);
2865 let wait = self.repository_wait();
2866 let outcome = wait
2867 .scope(agent::invoke(
2868 spec,
2869 &mut seat,
2870 &Invocation {
2871 coordination: self.state.config.coordination.as_ref(),
2872 fetched_base: None,
2873 cwd,
2874 prompt: &prompt,
2875 timeout,
2876 allow_write: false,
2877 sessions: false,
2878 artifacts,
2879 stem: &if n == 0 {
2880 "advise-synthesis".to_owned()
2881 } else {
2882 format!("advise-synthesis-{}", spec.id)
2883 },
2884 run: run_id,
2885 node: "advise",
2886 cache_dir: None,
2887 attachments: &[],
2888 writable: &[],
2889 },
2890 ))
2891 .await;
2892 if outcome.is_ok() {
2893 self.state.seats.insert(seat.key.clone(), seat);
2894 }
2895 let advance = agent::chain_advances(&outcome);
2896 last = Some(outcome);
2897 if !advance {
2898 break;
2899 }
2900 }
2901 let out = last.expect("a chain holds at least one agent")?;
2904 if !out.usable() {
2905 return Ok(None);
2906 }
2907 let text =
2908 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2909 Ok((!text.trim().is_empty()).then_some(text))
2910 }
2911
2912 async fn implement(&mut self) -> Result<()> {
2915 self.persist_candidates().await?;
2916 let run_id = self.state.id.clone();
2921 let prompts = self.state.config.prompts.clone();
2922 let todo: Vec<usize> = self
2923 .state
2924 .candidates
2925 .iter()
2926 .enumerate()
2927 .filter(|(_, c)| {
2928 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2929 && c.failed.is_none()
2930 && !c.empty
2931 })
2932 .map(|(i, _)| i)
2933 .collect();
2934 if todo.is_empty() {
2935 return self.after_implement();
2936 }
2937 self.state.status = RunStatus::Implementing;
2938
2939 let language = self.state.config.graph.language.clone();
2940 let github_language = self.state.config.graph.github_language.clone();
2941 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2942 let sessions = self.state.config.graph.sessions;
2943 let artifacts = agent::artifacts_dir(&self.state.dir());
2944 let brief = self
2948 .state
2949 .advice
2950 .as_ref()
2951 .and_then(|a| a.synthesis.as_deref())
2952 .map(str::to_owned);
2953 let attachments = self.state.attachments.clone();
2954
2955 let mut jobs = Vec::new();
2956 for &i in &todo {
2957 let (index, label, worktree) = {
2958 let c = &self.state.candidates[i];
2959 (c.index, c.label, c.worktree.clone())
2960 };
2961 let spec = self.roles.implementers[index].clone();
2962 let seat_key = format!("impl-{label}");
2963 let seat = self.seat(&seat_key, &spec.id);
2964 let instruction = seeded_instruction(&self.state);
2965 jobs.push(SeatJob {
2966 spec,
2967 seat,
2968 prompt: prompt::implement_in(
2969 &instruction,
2970 &worktree.to_string_lossy(),
2971 &language,
2972 &github_language,
2973 brief.as_deref(),
2974 &attachments,
2975 ),
2976 cwd: worktree,
2977 timeout,
2978 allow_write: true,
2979 sessions,
2980 artifacts: artifacts.clone(),
2981 stem: format!("impl-{label}"),
2982 handover: None,
2983 });
2984 }
2985
2986 self.state.event(
2987 "implement",
2988 format!("{} candidates in parallel", jobs.len()),
2989 );
2990 let mut sent = jobs.clone();
2996 let cache = self.state.config.cache_dir();
2997 let ctx = WaveCtx {
2998 carry_seats: false,
2999 pauses: Some((&self.pause, &self.interrupt)),
3000 run: &run_id,
3001 node: "implement",
3002 prompts: &prompts,
3003 cache: cache.as_deref(),
3004 round: None,
3005 };
3006 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3007 if !wave_blocked(&results) {
3008 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3009 .await;
3010 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3011 .await?;
3012 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3013 .await;
3014 }
3015 if !results.is_empty()
3018 && results
3019 .iter()
3020 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3021 {
3022 let losses: Vec<_> = results
3023 .iter()
3024 .map(|(_, seat, out)| {
3025 let AgentOutcome::Quota(o) = out else {
3026 unreachable!()
3027 };
3028 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3029 })
3030 .collect();
3031 self.state.quota.extend(losses.clone());
3032 for (_, seat, _) in results {
3033 self.state.seats.insert(seat.key.clone(), seat);
3034 }
3035 for &i in &todo {
3036 let label = self.state.candidates[i].label;
3037 self.state.interrupted_implementations.insert(label);
3038 self.state
3039 .candidate_preservation
3040 .insert(label, "machine outage awaiting persistence".into());
3041 }
3042 block_machine_outage(&mut self.state, "implement", &losses);
3045 self.state.save()?;
3046 self.persist_candidates().await?;
3047 for &i in &todo {
3048 self.state
3049 .interrupted_implementations
3050 .insert(self.state.candidates[i].label);
3051 }
3052 return self.stop_for_resource();
3053 }
3054 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3055
3056 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3057 && results
3058 .iter()
3059 .any(|(_, _, out)| FailClass::of(out).is_some());
3060 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3061 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3062 if self
3066 .state
3067 .candidate_preservation
3068 .contains_key(&self.state.candidates[i].label)
3069 {
3070 self.state.seats.insert(seat.key.clone(), seat);
3071 }
3072 continue;
3073 }
3074 let seat_key = seat.key.clone();
3075 let agent = seat.agent.clone();
3084 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3085 self.state.seats.insert(seat.key.clone(), seat);
3086 let label = self.state.candidates[i].label;
3087 self.state.interrupted_implementations.remove(&label);
3088
3089 let (summary, duration, failed, verified_claim) = match out {
3090 AgentOutcome::Ok(o) => {
3091 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3092 let failed = (!o.usable()).then(|| {
3093 if o.timed_out {
3094 "agent timed out".to_owned()
3095 } else {
3096 format!(
3097 "agent exited with {}",
3098 crate::notices::exit_status(o.exit_code)
3099 )
3100 }
3101 });
3102 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3103 (text, o.duration_ms, failed, verified_claim)
3104 }
3105 AgentOutcome::Dropped(o) => {
3111 let why = o
3112 .dropped
3113 .as_ref()
3114 .map(|d| d.why.as_str())
3115 .unwrap_or("the CLI ended the stream without delivering its answer");
3116 (
3117 String::new(),
3118 o.duration_ms,
3119 Some(format!("the CLI dropped the stream ({why})")),
3120 None,
3121 )
3122 }
3123 AgentOutcome::Quota(o) => {
3124 self.state.quota.push(QuotaLoss::from_report(
3125 seat_key,
3126 "implement",
3127 o.quota.as_ref(),
3128 ));
3129 (
3130 String::new(),
3131 o.duration_ms,
3132 Some("rate limited (quota); produced no change".to_owned()),
3133 None,
3134 )
3135 }
3136 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3137 (String::new(), 0, Some(e), None)
3138 }
3139 };
3140
3141 let c = &mut self.state.candidates[i];
3144 if !exhausted_the_fallback_chain {
3145 c.agent = agent;
3146 }
3147 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3148 c.duration_ms = duration;
3149 c.failed = failed;
3150 c.verified_noop = verified_claim;
3151 self.state
3152 .candidate_preservation
3153 .insert(label, "awaiting persistence".to_owned());
3154 }
3155 self.state.save()?;
3156 self.persist_candidates().await?;
3157
3158 if resource_blocked {
3159 return self.stop_for_resource();
3160 }
3161 if selected_failed {
3162 self.state.status = RunStatus::Blocked;
3163 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3164 self.state.save()?;
3165 bail!(
3166 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3167 );
3168 }
3169 self.after_implement()
3170 }
3171
3172 async fn persist_candidates(&mut self) -> Result<()> {
3174 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3175 for label in labels {
3176 let i = self
3177 .state
3178 .candidates
3179 .iter()
3180 .position(|c| c.label == label)
3181 .context("pending candidate is missing")?;
3182 let c = self.state.candidates[i].clone();
3183 let interrupted = self.state.interrupted_implementations.contains(&label);
3184 let result = async {
3185 let wait = crate::repo_lock::WaitPolicy::new(
3186 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3187 || false,
3188 );
3189 let _coordination =
3190 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3191 if let Err(e) = git::snapshot_uncommitted(
3194 &c.worktree,
3195 &self
3196 .state
3197 .dir()
3198 .join("artifacts")
3199 .join(format!("candidate-{label}-recovery")),
3200 )
3201 .await
3202 {
3203 self.state.event(
3204 "preserve",
3205 format!("candidate {label} snapshot incomplete: {e:#}"),
3206 );
3207 }
3208 let rescued = git::rescue_commit(
3209 &c.worktree,
3210 &format!("magi: candidate {label} (uncommitted work)"),
3211 )
3212 .await?;
3213 self.state.note_withheld("implement", &rescued.withheld);
3214 let base = &self.state.base_commit;
3215 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3216 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3217 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3218 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3219 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3220 if c.empty && commits > 0 && !patch.trim().is_empty()
3221 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3222 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3223 || self.state.tally.is_some())
3224 {
3225 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3229 }
3230 let c = &mut self.state.candidates[i];
3231 c.commits = commits;
3232 c.stat = stat;
3233 c.files = files;
3234 let no_changes = commits == 0 || patch.trim().is_empty();
3235 c.empty = no_changes && !interrupted;
3236 if interrupted && no_changes {
3237 c.failed = None;
3238 c.verified_noop = None;
3239 }
3240 if !c.empty {
3241 c.failed = None;
3242 c.verified_noop = None;
3243 }
3244 self.state.event(
3245 "preserve",
3246 format!("candidate {label}: {files} files, {commits} commits"),
3247 );
3248 Ok::<_, anyhow::Error>(())
3249 }
3250 .await;
3251 match result {
3252 Ok(()) => {
3253 self.state.candidate_preservation.remove(&label);
3254 if !interrupted || self.state.candidates[i].files > 0 {
3255 self.state.interrupted_implementations.remove(&label);
3256 }
3257 }
3258 Err(e) => {
3259 let reason = format!("candidate {label} preservation failed: {e:#}");
3260 self.state.event("preserve", &reason);
3261 self.state.candidate_preservation.insert(label, reason);
3262 }
3263 }
3264 if self.state.candidate_preservation.is_empty()
3265 && self.state.status == RunStatus::Blocked
3266 && self.state.candidates.iter().all(|c| c.failed.is_none())
3267 {
3268 self.state.status = RunStatus::Implementing;
3272 self.state.event(
3273 "preserve",
3274 "candidate persistence recovered; implementation can continue",
3275 );
3276 }
3277 self.state.save()?;
3278 }
3279 if !self.state.candidate_preservation.is_empty() {
3280 self.state.status = RunStatus::Blocked;
3281 self.state.save()?;
3282 bail!(
3283 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3284 );
3285 }
3286 Ok(())
3287 }
3288
3289 async fn resume_undelivered(
3317 &mut self,
3318 results: &mut [(usize, SeatState, AgentOutcome)],
3319 sent: &[SeatJob],
3320 prompts: &Prompts,
3321 run_id: &str,
3322 ) {
3323 for (wi, seat, out) in results.iter_mut() {
3324 let Some(dropped) = (match &*out {
3325 AgentOutcome::Dropped(o) => o.dropped.clone(),
3326 _ => None,
3327 }) else {
3328 continue;
3329 };
3330 let Some(job) = sent.get(*wi) else { continue };
3331 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3333 self.state.event(
3334 "implement",
3335 format!(
3336 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3337 work is in the tree",
3338 seat.key, dropped.output_tokens, dropped.why
3339 ),
3340 );
3341 continue;
3342 }
3343 if !has_context(&job.spec, seat, job.sessions) {
3351 self.state.event(
3352 "implement",
3353 format!(
3354 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3355 is no session left to resume",
3356 seat.key, dropped.output_tokens, dropped.why
3357 ),
3358 );
3359 continue;
3360 }
3361 self.state.event(
3362 "implement",
3363 format!(
3364 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3365 conversation",
3366 seat.key, dropped.output_tokens, dropped.why
3367 ),
3368 );
3369 let mut retry = job.clone();
3370 retry.seat = seat.clone();
3371 retry.prompt = prompt::resume_after_drop(&dropped.why);
3372 retry.timeout = retry_budget(job.timeout, true);
3373 retry.stem = format!("{}-resume", job.stem);
3374 let cache = self.state.config.cache_dir();
3375 let ctx = WaveCtx {
3376 carry_seats: false,
3377 pauses: Some((&self.pause, &self.interrupt)),
3378 run: run_id,
3379 node: "implement",
3380 prompts,
3381 cache: cache.as_deref(),
3382 round: None,
3383 };
3384 let (resumed_seat, resumed) =
3385 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3386 *seat = resumed_seat;
3387 *out = resumed;
3388 }
3389 }
3390
3391 async fn resume_seat_handovers(
3456 &mut self,
3457 results: &mut [(usize, SeatState, AgentOutcome)],
3458 sent: &mut [SeatJob],
3459 prompts: &Prompts,
3460 run_id: &str,
3461 ) -> Result<()> {
3462 let instruction = seeded_instruction(&self.state);
3463 let language = self.state.config.graph.language.clone();
3464 let github_language = self.state.config.graph.github_language.clone();
3465 let brief = self
3466 .state
3467 .advice
3468 .as_ref()
3469 .and_then(|a| a.synthesis.as_deref())
3470 .map(str::to_owned);
3471 let attachments = self.state.attachments.clone();
3472 for (wi, seat, out) in results.iter_mut() {
3473 let others: BTreeSet<String> = sent
3476 .iter()
3477 .enumerate()
3478 .filter(|(j, _)| j != wi)
3479 .map(|(_, j)| j.spec.id.clone())
3480 .collect();
3481 let Some(job) = sent.get_mut(*wi) else {
3482 continue;
3483 };
3484 let start = self
3489 .roles
3490 .implementer_roster
3491 .iter()
3492 .position(|s| s.id == job.spec.id)
3493 .unwrap_or(0);
3494 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3495 let mut fallback_attempt = 0usize;
3496 let mut prev: Option<FailClass> = None;
3497 while let Some(cur) = FailClass::of(&*out) {
3498 if !should_hand_over(prev.as_ref(), &cur) {
3499 break;
3500 }
3501 let Some(next) =
3502 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3503 .cloned()
3504 else {
3505 break;
3506 };
3507 tried.insert(next.id.clone());
3508 fallback_attempt += 1;
3509
3510 let coordination = self.coordinate(&job.cwd).await?;
3511 if let Ok(r) = git::rescue_commit(
3512 &job.cwd,
3513 &format!(
3514 "magi: candidate {} (uncommitted work before {} fallback)",
3515 seat.key,
3516 if cur == FailClass::Quota {
3517 "quota"
3518 } else {
3519 "handover"
3520 }
3521 ),
3522 )
3523 .await
3524 {
3525 self.state.note_withheld("implement", &r.withheld);
3526 }
3527 drop(coordination);
3528
3529 record_handover(
3530 &mut self.state,
3531 "implement",
3532 &seat.key,
3533 &seat.agent,
3534 &next.id,
3535 &cur,
3536 &fail_reason(&*out),
3537 );
3538 prev = Some(cur.clone());
3539
3540 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3541 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3542 job.spec = next.clone();
3550 let mut retry = job.clone();
3551 retry.seat = new_seat;
3552 retry.prompt = prompt::implement_in(
3553 &instruction,
3554 &job.cwd.to_string_lossy(),
3555 &language,
3556 &github_language,
3557 brief.as_deref(),
3558 &attachments,
3559 );
3560 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3561 let cache = self.state.config.cache_dir();
3562 let ctx = WaveCtx {
3563 carry_seats: false,
3564 pauses: Some((&self.pause, &self.interrupt)),
3565 run: run_id,
3566 node: "implement",
3567 prompts,
3568 cache: cache.as_deref(),
3569 round: None,
3570 };
3571 let (fallback_seat, fallback_out) = run_one(
3572 retry,
3573 Arc::clone(&self.sem),
3574 &ctx,
3575 &mut self.state,
3576 fallback_attempt,
3577 )
3578 .await;
3579 *seat = fallback_seat;
3580 *out = fallback_out;
3581 }
3582 }
3583 Ok(())
3584 }
3585
3586 async fn resume_unconfirmed_commands(
3610 &mut self,
3611 results: &mut [(usize, SeatState, AgentOutcome)],
3612 sent: &[SeatJob],
3613 prompts: &Prompts,
3614 run_id: &str,
3615 ) {
3616 for (wi, seat, out) in results.iter_mut() {
3617 let AgentOutcome::Ok(o) = &*out else {
3618 continue;
3619 };
3620 if !has_unconfirmed_command(&o.commands) {
3621 continue;
3622 }
3623 let Some(job) = sent.get(*wi) else { continue };
3624 if !has_context(&job.spec, seat, job.sessions) {
3625 self.state.event(
3626 "implement",
3627 format!(
3628 "{}: the reply named a command whose own CLI never confirmed the exit \
3629 status of, but there is no session left to resume",
3630 seat.key
3631 ),
3632 );
3633 continue;
3634 }
3635 self.state.event(
3636 "implement",
3637 format!(
3638 "{}: the reply named a command whose own CLI never confirmed the exit \
3639 status of; resuming the conversation",
3640 seat.key
3641 ),
3642 );
3643 let mut retry = job.clone();
3644 retry.seat = seat.clone();
3645 retry.prompt = prompt::resume_incomplete(
3646 "a command in your last reply had no confirmed exit status",
3647 );
3648 retry.timeout = retry_budget(job.timeout, true);
3649 retry.stem = format!("{}-confirm", job.stem);
3650 let cache = self.state.config.cache_dir();
3651 let ctx = WaveCtx {
3652 carry_seats: false,
3653 pauses: Some((&self.pause, &self.interrupt)),
3654 run: run_id,
3655 node: "implement",
3656 prompts,
3657 cache: cache.as_deref(),
3658 round: None,
3659 };
3660 let (resumed_seat, resumed) =
3661 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3662 *seat = resumed_seat;
3663 *out = resumed;
3664 }
3665 }
3666
3667 async fn continue_fix_report(
3688 &mut self,
3689 mut seat: SeatState,
3690 parse_err: String,
3691 job: &SeatJob,
3692 prompts: &Prompts,
3693 run_id: &str,
3694 round: usize,
3695 ) -> (
3696 SeatState,
3697 Option<FixReport>,
3698 Option<String>,
3699 ContinuationRecord,
3700 ) {
3701 let mut last_err = parse_err;
3702 let mut cumulative_wait_ms = 0u64;
3703 let mut attempts = 0usize;
3704 loop {
3705 if !has_context(&job.spec, &seat, job.sessions) {
3706 self.state.event(
3707 "fix",
3708 format!(
3709 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3710 session left to resume into"
3711 ),
3712 );
3713 let outcome = if attempts == 0 {
3714 ContinuationOutcome::NoSession
3715 } else {
3716 ContinuationOutcome::Exhausted
3717 };
3718 return (
3719 seat,
3720 None,
3721 Some(format!("unparsable fix report: {last_err}")),
3722 ContinuationRecord {
3723 attempts,
3724 cumulative_wait_ms,
3725 outcome,
3726 },
3727 );
3728 }
3729 if attempts >= MAX_FIX_CONTINUATIONS {
3730 self.state.event(
3731 "fix",
3732 format!(
3733 "round {round}: fixer's reply still had no adoption report after \
3734 {attempts} continuation(s) ({last_err}); giving up"
3735 ),
3736 );
3737 return (
3738 seat,
3739 None,
3740 Some(format!(
3741 "unparsable fix report after {attempts} continuation(s): {last_err}"
3742 )),
3743 ContinuationRecord {
3744 attempts,
3745 cumulative_wait_ms,
3746 outcome: ContinuationOutcome::Exhausted,
3747 },
3748 );
3749 }
3750 attempts += 1;
3751 self.state.event(
3752 "fix",
3753 format!(
3754 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3755 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3756 ),
3757 );
3758 let mut retry = job.clone();
3759 retry.seat = seat.clone();
3760 retry.prompt = prompt::resume_incomplete(&last_err);
3761 retry.timeout = retry_budget(job.timeout, true);
3762 retry.stem = format!("{}-continue{attempts}", job.stem);
3763 let cache = self.state.config.cache_dir();
3764 let ctx = WaveCtx {
3765 carry_seats: false,
3766 pauses: Some((&self.pause, &self.interrupt)),
3767 run: run_id,
3768 node: "fix",
3769 prompts,
3770 cache: cache.as_deref(),
3771 round: Some(round),
3772 };
3773 let (resumed_seat, resumed_out) = run_one(
3774 retry,
3775 Arc::clone(&self.sem),
3776 &ctx,
3777 &mut self.state,
3778 attempts,
3779 )
3780 .await;
3781 seat = resumed_seat;
3782 match resumed_out {
3783 AgentOutcome::Ok(o) => {
3784 cumulative_wait_ms += o.duration_ms;
3785 match verdict::extract_json::<FixReport>(&o.text) {
3786 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3787 self.state.event(
3788 "fix",
3789 format!(
3790 "round {round}: fixer's adoption report recovered after \
3791 {attempts} continuation(s)"
3792 ),
3793 );
3794 return (
3795 seat,
3796 Some(report),
3797 None,
3798 ContinuationRecord {
3799 attempts,
3800 cumulative_wait_ms,
3801 outcome: ContinuationOutcome::Resumed,
3802 },
3803 );
3804 }
3805 Ok(_) => {
3813 last_err = "the reply parsed, but it reported a command whose own CLI \
3814 never confirmed an exit status"
3815 .to_owned();
3816 }
3817 Err(e) => last_err = e.to_string(),
3818 }
3819 }
3820 AgentOutcome::Quota(o) => {
3821 cumulative_wait_ms += o.duration_ms;
3822 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3823 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3824 self.state.quota.push(loss);
3825 self.state.event(
3826 "fix",
3827 format!(
3828 "round {round}: continuation rate limited (quota); not retrying now"
3829 ),
3830 );
3831 return (
3832 seat,
3833 None,
3834 Some("rate limited (quota) while recovering the fix report".to_owned()),
3835 ContinuationRecord {
3836 attempts,
3837 cumulative_wait_ms,
3838 outcome: ContinuationOutcome::QuotaLost,
3839 },
3840 );
3841 }
3842 AgentOutcome::Dropped(o) => {
3843 cumulative_wait_ms += o.duration_ms;
3844 let why = o
3845 .dropped
3846 .as_ref()
3847 .map(|d| d.why.as_str())
3848 .unwrap_or("the CLI ended the stream without delivering its answer");
3849 last_err = format!("the CLI dropped the stream ({why})");
3850 }
3851 AgentOutcome::Failed(e) => last_err = e,
3852 AgentOutcome::ResourceBlocked(e) => {
3855 return (
3856 seat,
3857 None,
3858 Some(format!("the continuation was not started: {e}")),
3859 ContinuationRecord {
3860 attempts: attempts.saturating_sub(1),
3861 cumulative_wait_ms,
3862 outcome: ContinuationOutcome::NoSession,
3863 },
3864 );
3865 }
3866 }
3867 }
3868 }
3869
3870 fn after_implement(&mut self) -> Result<()> {
3871 if !self.state.candidate_preservation.is_empty() {
3872 bail!("candidate preservation incomplete; judging is unsafe");
3873 }
3874 if self.state.leaks.is_empty() {
3876 let cfg = self.state.config.blind.clone();
3877 let mut leaks = Vec::new();
3878 for c in &self.state.candidates {
3879 let Some(patch) =
3880 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3881 else {
3882 continue;
3883 };
3884 leaks.extend(blind::scan(
3885 &format!("candidate {} patch", c.label),
3886 &patch,
3887 &cfg.vendor_tokens,
3888 ));
3889 }
3890 if !leaks.is_empty() {
3891 let summary = leaks
3892 .iter()
3893 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3894 .collect::<Vec<_>>()
3895 .join(", ");
3896 match cfg.on_leak {
3897 LeakPolicy::Fail => {
3898 self.state.status = RunStatus::Failed;
3899 self.state
3900 .event("blind", format!("vendor text in a patch: {summary}"));
3901 self.state.leaks = leaks;
3902 self.state.save()?;
3903 self.settle_questions();
3904 bail!(
3905 "blind.on_leak = \"fail\" and vendor text reached a \
3906 judged patch: {summary}"
3907 );
3908 }
3909 LeakPolicy::Redact => self.state.event(
3910 "blind",
3911 format!("redacting vendor text for judging: {summary}"),
3912 ),
3913 LeakPolicy::Warn => self.state.event(
3914 "blind",
3915 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3916 ),
3917 }
3918 self.state.leaks = leaks;
3919 }
3920 }
3921
3922 if self.state.viable().is_empty() {
3923 if self.state.all_candidates_verified_noop() {
3924 self.state.status = RunStatus::VerifiedNoop;
3935 self.state.save()?;
3936 self.settle_questions();
3937 return Ok(());
3938 }
3939 self.state.status = RunStatus::Failed;
3940 self.state.save()?;
3941 self.settle_questions();
3942 bail!("no candidate produced a change; nothing to judge");
3943 }
3944 self.state.status = RunStatus::Judging;
3945 self.state.save()?;
3946 Ok(())
3947 }
3948
3949 async fn judge(&mut self) -> Result<()> {
3952 let run_id = self.state.id.clone();
3957 let prompts = self.state.config.prompts.clone();
3958 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3959 return Ok(());
3960 }
3961 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3962 if viable.len() == 1 {
3963 self.state.judge_skipped = true;
3970 self.state.event(
3971 "judge",
3972 format!(
3973 "only candidate {} produced a change; judging skipped",
3974 viable[0].label
3975 ),
3976 );
3977 self.state.save()?;
3978 return Ok(());
3979 }
3980 self.state.status = RunStatus::Judging;
3981
3982 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983 let language = self.state.config.graph.language.clone();
3984 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985 let sessions = self.state.config.graph.sessions;
3986 let artifacts = agent::artifacts_dir(&self.state.dir());
3987 let root = self.state.worktree_root();
3988 let base_short = short(&self.state.base_commit);
3989
3990 let mut jobs = Vec::new();
3991 let mut orders = Vec::new();
3992 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3993 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3994 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3995 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3996 let seat_key = format!("judge-{}", j + 1);
3997 let seat = self.seat(&seat_key, &spec.id);
3998 jobs.push(SeatJob {
3999 prompt: prompt::judge(
4000 &self.state.instruction,
4001 &views,
4002 self.roles.judges.len(),
4003 &base_short,
4004 &language,
4005 ),
4006 spec,
4007 seat,
4008 cwd: root.join(format!("judge-{}", j + 1)),
4009 timeout,
4010 allow_write: false,
4011 sessions,
4012 artifacts: artifacts.clone(),
4013 stem: format!("judge-{}", j + 1),
4014 handover: None,
4015 });
4016 }
4017
4018 self.state.event(
4019 "judge",
4020 format!(
4021 "{} judges ranking {} candidates blind",
4022 jobs.len(),
4023 viable.len()
4024 ),
4025 );
4026 let labels_for_check = labels.clone();
4027 let mut quota_losses = Vec::new();
4028 let cache = self.state.config.cache_dir();
4029 let ctx = WaveCtx {
4030 carry_seats: false,
4031 pauses: Some((&self.pause, &self.interrupt)),
4032 run: &run_id,
4033 node: "judge",
4034 prompts: &prompts,
4035 cache: cache.as_deref(),
4036 round: None,
4037 };
4038 let results = ask_json_wave::<Ranking>(
4039 jobs,
4040 Arc::clone(&self.sem),
4041 self.state.config.graph.retries,
4042 &self.roles.judge_roster,
4043 &ctx,
4044 &mut quota_losses,
4045 &mut self.state,
4046 &move |r: &Ranking| r.validate(&labels_for_check),
4047 )
4048 .await;
4049 if self.state.resource_block.is_some() {
4050 return self.stop_for_resource();
4051 }
4052 self.state.quota.extend(quota_losses);
4053
4054 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4055 let agent_id = seat.agent.clone();
4056 self.state.seats.insert(seat.key.clone(), seat);
4057 let mut record = Judgement {
4058 judge: j + 1,
4059 seat: format!("judge-{}", j + 1),
4060 agent: agent_id,
4061 ranking: Vec::new(),
4062 reasons: BTreeMap::new(),
4063 confidence: None,
4064 strengths: BTreeMap::new(),
4065 order: orders[j].clone(),
4066 failed: None,
4067 duration_ms: 0,
4068 };
4069 match res {
4070 Ok((ranking, out)) => {
4071 record.ranking = ranking.normalized();
4072 record.reasons = ranking.reasons;
4073 record.strengths = ranking.strengths;
4074 record.confidence = ranking.confidence;
4075 record.duration_ms = out.duration_ms;
4076 self.state.event(
4077 "judge",
4078 format!(
4079 "judge {} ranked {}",
4080 j + 1,
4081 record.ranking.iter().collect::<String>()
4082 ),
4083 );
4084 }
4085 Err(e) => {
4086 record.failed = Some(e.to_string());
4087 self.state
4088 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4089 }
4090 }
4091 self.state.judgements.push(record);
4092 self.state.save()?;
4093 }
4094 Ok(())
4095 }
4096
4097 async fn deliberate(&mut self) -> Result<()> {
4100 if self.hold_base_sync().await? {
4101 return Ok(());
4102 }
4103 let run_id = self.state.id.clone();
4108 let prompts = self.state.config.prompts.clone();
4109 let rounds = self.state.config.graph.deliberate_rounds;
4110 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4112 return Ok(());
4113 }
4114 let tops: Vec<char> = self
4115 .state
4116 .judgements
4117 .iter()
4118 .filter_map(|j| j.ranking.first().copied())
4119 .collect();
4120 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4121 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4122 self.state.event(
4123 "deliberate",
4124 format!("judges agreed on {} outright; no deliberation", tops[0]),
4125 );
4126 }
4127 self.state.status = RunStatus::Voting;
4128 self.state.save()?;
4129 return Ok(());
4130 }
4131
4132 self.state.status = RunStatus::Deliberating;
4133 self.state.event(
4134 "deliberate",
4135 format!(
4136 "split: first choices were {} — opening {rounds} round(s)",
4137 tops.iter().collect::<String>()
4138 ),
4139 );
4140
4141 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4142 let language = self.state.config.graph.language.clone();
4143 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4144 let sessions = self.state.config.graph.sessions;
4145 let artifacts = agent::artifacts_dir(&self.state.dir());
4146 let root = self.state.worktree_root();
4147 let base_short = short(&self.state.base_commit);
4148
4149 for round in self.state.deliberation.len() + 1..=rounds {
4153 let mut turns: Vec<DeliberationTurn> = Vec::new();
4154 let mut round_losses = Vec::new();
4155 let mut asked = 0;
4156 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4157 if self.state.judgements[j].failed.is_some() {
4158 continue;
4159 }
4160 let seat_key = format!("judge-{}", j + 1);
4161 let spec = self.occupant(&seat_key, spec);
4162 let mut seat = self.seat(&seat_key, &spec.id);
4163 let transcript = self.transcript(&turns, j);
4164 let build = |context: Option<&str>| {
4165 prompt::deliberate(
4166 &self.state.instruction,
4167 context,
4168 &transcript,
4169 round,
4170 rounds,
4171 &language,
4172 )
4173 };
4174 let block = self.candidate_block(&viable, &base_short);
4175 let full = build(Some(&block));
4176 let text = if has_context(&spec, &seat, sessions) {
4177 build(None)
4178 } else {
4179 full.clone()
4180 };
4181 let job = SeatJob {
4182 spec,
4183 seat: seat.clone(),
4184 prompt: text,
4185 cwd: root.join(format!("judge-{}", j + 1)),
4186 timeout,
4187 allow_write: false,
4188 sessions,
4189 artifacts: artifacts.clone(),
4190 stem: format!("delib-{round}-judge-{}", j + 1),
4191 handover: Some(full),
4192 };
4193 let cache = self.state.config.cache_dir();
4194 let ctx = WaveCtx {
4195 carry_seats: false,
4196 pauses: Some((&self.pause, &self.interrupt)),
4197 run: &run_id,
4198 node: "deliberate",
4199 prompts: &prompts,
4200 cache: cache.as_deref(),
4201 round: None,
4202 };
4203 let mut losses = Vec::new();
4207 asked += 1;
4208 let mut results = ask_wave_with::<String>(
4209 vec![job],
4210 Arc::clone(&self.sem),
4211 0,
4212 &self.roles.judge_roster,
4213 &ctx,
4214 &mut losses,
4215 &mut self.state,
4216 &|text: &str| {
4217 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4218 },
4219 )
4220 .await;
4221 if self.state.resource_block.is_some() {
4222 return self.stop_for_resource();
4223 }
4224 round_losses.extend(losses.iter().cloned());
4225 self.state.quota.extend(losses);
4226 let (updated, res, _) = results.pop().expect("one job in, one result out");
4227 seat = updated;
4228 let agent_id = seat.agent.clone();
4229 self.state.seats.insert(seat.key.clone(), seat);
4230 let body = match res {
4231 Ok((body, _)) => body,
4232 Err(e) => {
4235 self.state
4236 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4237 continue;
4238 }
4239 };
4240 let tentative = verdict::extract_json::<Position>(&body)
4241 .ok()
4242 .and_then(|p| p.tentative)
4243 .and_then(|s| s.trim().chars().next())
4244 .map(|c| c.to_ascii_uppercase());
4245 self.state.event(
4246 "deliberate",
4247 format!(
4248 "round {round}: judge {} now favours {}",
4249 j + 1,
4250 tentative.map_or("—".to_owned(), |c| c.to_string())
4251 ),
4252 );
4253 turns.push(DeliberationTurn {
4254 judge: j + 1,
4255 agent: agent_id,
4256 body: blind::sanitize_prose(&body, &self.state.config.blind),
4257 tentative,
4258 });
4259 }
4260 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4261 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4262 return self.stop_for_resource();
4263 }
4264 self.state
4265 .deliberation
4266 .push(DeliberationRound { round, turns });
4267 self.state.save()?;
4268 }
4269
4270 self.state.status = RunStatus::Voting;
4271 self.state.save()?;
4272 Ok(())
4273 }
4274
4275 async fn vote(&mut self) -> Result<()> {
4278 if self.hold_base_sync().await? {
4279 return Ok(());
4280 }
4281 let run_id = self.state.id.clone();
4286 let prompts = self.state.config.prompts.clone();
4287 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4288 return Ok(());
4289 }
4290 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4291 if viable.len() == 1 {
4292 return Ok(());
4293 }
4294 self.state.status = RunStatus::Voting;
4295
4296 let language = self.state.config.graph.language.clone();
4297 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4298 let sessions = self.state.config.graph.sessions;
4299 let artifacts = agent::artifacts_dir(&self.state.dir());
4300 let root = self.state.worktree_root();
4301 let base_short = short(&self.state.base_commit);
4302 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4303
4304 let mut jobs = Vec::new();
4305 let mut seats_at = Vec::new();
4306 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4307 if self
4308 .state
4309 .judgements
4310 .get(j)
4311 .is_some_and(|r| r.failed.is_some())
4312 {
4313 continue;
4314 }
4315 let seat_key = format!("judge-{}", j + 1);
4316 let spec = self.occupant(&seat_key, spec);
4317 let seat = self.seat(&seat_key, &spec.id);
4318 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4319 let text = if has_context(&spec, &seat, sessions) {
4320 prompt::final_vote(&viable, &language)
4321 } else {
4322 full.clone()
4323 };
4324 jobs.push(SeatJob {
4325 spec,
4326 seat,
4327 prompt: text,
4328 cwd: root.join(format!("judge-{}", j + 1)),
4329 timeout,
4330 allow_write: false,
4331 sessions,
4332 artifacts: artifacts.clone(),
4333 stem: format!("vote-judge-{}", j + 1),
4334 handover: Some(full),
4335 });
4336 seats_at.push(j);
4337 }
4338
4339 self.state.event(
4340 "vote",
4341 format!(
4342 "collecting {} final votes one by one, privately",
4343 jobs.len()
4344 ),
4345 );
4346 let allowed = viable.clone();
4347 let mut quota_losses = Vec::new();
4348 let cache = self.state.config.cache_dir();
4349 let ctx = WaveCtx {
4350 carry_seats: false,
4351 pauses: Some((&self.pause, &self.interrupt)),
4352 run: &run_id,
4353 node: "vote",
4354 prompts: &prompts,
4355 cache: cache.as_deref(),
4356 round: None,
4357 };
4358 let results = ask_json_wave::<FinalVote>(
4359 jobs,
4360 Arc::clone(&self.sem),
4361 self.state.config.graph.retries,
4362 &self.roles.judge_roster,
4363 &ctx,
4364 &mut quota_losses,
4365 &mut self.state,
4366 &move |v: &FinalVote| match v.label() {
4367 Some(c) if allowed.contains(&c) => Ok(()),
4368 other => bail!("vote {other:?} is not one of {allowed:?}"),
4369 },
4370 )
4371 .await;
4372 if self.state.resource_block.is_some() {
4373 return self.stop_for_resource();
4374 }
4375 self.state.quota.extend(quota_losses);
4376
4377 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4378 let agent_id = seat.agent.clone();
4379 self.state.seats.insert(seat.key.clone(), seat);
4380 let initial = self
4381 .state
4382 .judgements
4383 .get(j)
4384 .and_then(|r| r.ranking.first().copied());
4385 let mut record = VoteRecord {
4386 judge: j + 1,
4387 agent: agent_id,
4388 vote: None,
4389 reason: String::new(),
4390 changed: false,
4391 };
4392 match res {
4393 Ok((v, _)) => {
4394 record.vote = v.label();
4395 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4396 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4397 self.state.event(
4398 "vote",
4399 format!(
4400 "judge {} voted {}{}",
4401 j + 1,
4402 record.vote.unwrap_or('?'),
4403 if record.changed { " (changed)" } else { "" }
4404 ),
4405 );
4406 }
4407 Err(e) => {
4408 self.state
4409 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4410 }
4411 }
4412 self.state.votes.push(record);
4413 self.state.save()?;
4414 }
4415 Ok(())
4416 }
4417
4418 fn tally(&mut self) -> Result<()> {
4421 if self.state.tally.is_some() {
4422 return Ok(());
4423 }
4424 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4425 let tops: Vec<char> = self
4426 .state
4427 .judgements
4428 .iter()
4429 .filter_map(|j| j.ranking.first().copied())
4430 .collect();
4431 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4432
4433 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4436 let mut cast: Vec<char> = Vec::new();
4437 for (i, j) in self.state.judgements.iter().enumerate() {
4438 let vote = self
4439 .state
4440 .votes
4441 .iter()
4442 .find(|v| v.judge == i + 1)
4443 .and_then(|v| v.vote)
4444 .or_else(|| j.ranking.first().copied());
4445 if let Some(v) = vote {
4446 *first_choice.entry(v).or_insert(0) += 1;
4447 cast.push(v);
4448 }
4449 }
4450
4451 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4452 for j in &self.state.judgements {
4453 let n = j.ranking.len();
4454 for (pos, label) in j.ranking.iter().enumerate() {
4455 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4456 }
4457 }
4458
4459 let best = first_choice.values().copied().max().unwrap_or(0);
4460 let mut leaders: Vec<char> = first_choice
4461 .iter()
4462 .filter(|(_, v)| **v == best)
4463 .map(|(k, _)| *k)
4464 .collect();
4465 let mut tie_break = None;
4466 if leaders.len() > 1 {
4467 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4468 let borda_leaders: Vec<char> = leaders
4469 .iter()
4470 .copied()
4471 .filter(|l| borda[l] == top_borda)
4472 .collect();
4473 tie_break = Some(if borda_leaders.len() == 1 {
4474 format!(
4475 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4476 leaders.len()
4477 )
4478 } else {
4479 format!(
4480 "{} way tie on both first-choice votes and Borda points, broken by label order",
4481 leaders.len()
4482 )
4483 });
4484 leaders = borda_leaders;
4485 leaders.sort_unstable();
4486 }
4487 let winner = *leaders
4488 .first()
4489 .or(viable.first())
4490 .context("no candidate to declare a winner from")?;
4491
4492 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4493 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4494 let deliberated = !self.state.deliberation.is_empty();
4495
4496 let quota_seats: std::collections::BTreeSet<&str> =
4500 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4501 let mut present = 0usize;
4502 for (i, j) in self.state.judgements.iter().enumerate() {
4503 if quota_seats.contains(j.seat.as_str()) {
4504 continue;
4505 }
4506 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4507 let voted = self
4508 .state
4509 .votes
4510 .iter()
4511 .any(|v| v.judge == i + 1 && v.vote.is_some());
4512 if ranked || voted {
4513 present += 1;
4514 }
4515 }
4516 let needs_quorum = viable.len() > 1;
4522 let judges_total = if needs_quorum {
4523 self.roles.judges.len()
4524 } else {
4525 0
4526 };
4527 let quorum = if needs_quorum {
4528 judges_total / 2 + 1
4529 } else {
4530 0
4531 };
4532 let met_quorum = !needs_quorum || present >= quorum;
4533 let uncontested = (!needs_quorum).then(|| {
4534 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4535 });
4536
4537 self.state.event(
4538 "tally",
4539 match &uncontested {
4540 Some(reason) => format!("winner {winner} — {reason}"),
4541 None => format!(
4542 "winner {winner} — votes {} | initial {} | {} changed | \
4543 {present}/{judges_total} judges{}",
4544 first_choice
4545 .iter()
4546 .map(|(k, v)| format!("{k}:{v}"))
4547 .collect::<Vec<_>>()
4548 .join(" "),
4549 if unanimous_initial {
4550 "unanimous"
4551 } else {
4552 "split"
4553 },
4554 changed_votes,
4555 if met_quorum {
4556 String::new()
4557 } else {
4558 format!(" — below quorum ({quorum} required)")
4559 },
4560 ),
4561 },
4562 );
4563 if !met_quorum {
4564 self.state.event(
4565 "stall",
4566 format!(
4567 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4568 the run stops here, resumable"
4569 ),
4570 );
4571 }
4572 self.state.tally = Some(Tally {
4573 first_choice,
4574 borda,
4575 winner,
4576 rankings: tops.len(),
4577 unanimous_initial,
4578 deliberated,
4579 changed_votes,
4580 unanimous_final,
4581 tie_break,
4582 judges: judges_total,
4583 present,
4584 quorum,
4585 met_quorum,
4586 uncontested,
4587 });
4588 self.state.status = if met_quorum {
4589 RunStatus::Reviewing
4590 } else {
4591 RunStatus::Stalled
4592 };
4593 self.state.save()?;
4594 Ok(())
4595 }
4596
4597 #[allow(clippy::too_many_lines)]
4618 async fn recover_stall(&mut self) -> Result<bool> {
4619 let run_id = self.state.id.clone();
4624 let prompts = self.state.config.prompts.clone();
4625 let quota_seats: BTreeSet<&str> =
4630 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4631 let absent: Vec<String> = self
4632 .state
4633 .judgements
4634 .iter()
4635 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4636 .map(|j| j.seat.clone())
4637 .collect();
4638 if absent.is_empty() {
4639 return Ok(false);
4640 }
4641 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4642 if viable.len() <= 1 {
4643 return Ok(false);
4644 }
4645 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4646 let language = self.state.config.graph.language.clone();
4647 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4648 let sessions = self.state.config.graph.sessions;
4649 let artifacts = agent::artifacts_dir(&self.state.dir());
4650 let root = self.state.worktree_root();
4651 let base_short = short(&self.state.base_commit);
4652 let candidates: Vec<Candidate> = viable.clone();
4653
4654 let mut positions: Vec<usize> = absent
4656 .iter()
4657 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4658 .collect();
4659 if positions.is_empty() {
4660 return Ok(false);
4661 }
4662 positions.sort_unstable();
4663 positions.dedup();
4664
4665 let mut judge_jobs = Vec::new();
4667 for &j in &positions {
4668 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4669 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4670 let seat_key = format!("judge-{}", j + 1);
4671 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4672 let seat = self.seat(&seat_key, &spec.id);
4673 judge_jobs.push(SeatJob {
4674 spec,
4675 seat,
4676 prompt: prompt::judge(
4677 &self.state.instruction,
4678 &views,
4679 self.roles.judges.len(),
4680 &base_short,
4681 &language,
4682 ),
4683 cwd: root.join(seat_key),
4684 timeout,
4685 allow_write: false,
4686 sessions,
4687 artifacts: artifacts.clone(),
4688 stem: format!("judge-{}-recover", j + 1),
4689 handover: None,
4690 });
4691 }
4692
4693 let labels_for_check = labels.clone();
4694 let mut judge_losses = Vec::new();
4695 let retries = self.state.config.graph.retries;
4696 let cache = self.state.config.cache_dir();
4697 let ctx = WaveCtx {
4698 carry_seats: false,
4699 pauses: Some((&self.pause, &self.interrupt)),
4700 run: &run_id,
4701 node: "judge",
4702 prompts: &prompts,
4703 cache: cache.as_deref(),
4704 round: None,
4705 };
4706 let results = ask_json_wave::<Ranking>(
4707 judge_jobs,
4708 Arc::clone(&self.sem),
4709 retries,
4710 &self.roles.judge_roster,
4711 &ctx,
4712 &mut judge_losses,
4713 &mut self.state,
4714 &move |r: &Ranking| r.validate(&labels_for_check),
4715 )
4716 .await;
4717 if self.state.resource_block.is_some() {
4718 self.stop_for_resource()?;
4719 return Ok(false);
4720 }
4721
4722 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4724 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4725 let agent_id = seat.agent.clone();
4726 self.state.seats.insert(seat.key.clone(), seat);
4727 let record = &mut self.state.judgements[j];
4728 match res {
4729 Ok((ranking, out)) => {
4730 record.agent = agent_id;
4731 record.ranking = ranking.normalized();
4732 record.reasons = ranking.reasons;
4733 record.strengths = ranking.strengths;
4734 record.confidence = ranking.confidence;
4735 record.failed = None;
4736 record.duration_ms = out.duration_ms;
4737 recovered.insert(j);
4738 self.state.event(
4739 "recover",
4740 format!("judge {} ranked again after the limit", j + 1),
4741 );
4742 }
4743 Err(e) => {
4744 self.state
4745 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4746 }
4747 }
4748 }
4749
4750 let mut vote_jobs = Vec::new();
4752 let mut vote_pos: Vec<usize> = Vec::new();
4753 for &j in &recovered {
4754 let seat_key = format!("judge-{}", j + 1);
4755 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4756 let seat = self.seat(&seat_key, &spec.id);
4757 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4758 let text = if has_context(&spec, &seat, sessions) {
4759 prompt::final_vote(&labels, &language)
4760 } else {
4761 full.clone()
4762 };
4763 vote_jobs.push(SeatJob {
4764 spec,
4765 seat,
4766 prompt: text,
4767 cwd: root.join(seat_key),
4768 timeout,
4769 allow_write: false,
4770 sessions,
4771 artifacts: artifacts.clone(),
4772 stem: format!("vote-judge-{}-recover", j + 1),
4773 handover: Some(full),
4774 });
4775 vote_pos.push(j);
4776 }
4777 let allowed = labels.clone();
4778 let mut vote_losses = Vec::new();
4779 let vote_retries = self.state.config.graph.retries;
4780 let vote_cache = self.state.config.cache_dir();
4781 let ctx = WaveCtx {
4782 carry_seats: false,
4783 pauses: Some((&self.pause, &self.interrupt)),
4784 run: &run_id,
4785 node: "vote",
4786 prompts: &prompts,
4787 cache: vote_cache.as_deref(),
4788 round: None,
4789 };
4790 let votes = ask_json_wave::<FinalVote>(
4791 vote_jobs,
4792 Arc::clone(&self.sem),
4793 vote_retries,
4794 &self.roles.judge_roster,
4795 &ctx,
4796 &mut vote_losses,
4797 &mut self.state,
4798 &move |v: &FinalVote| match v.label() {
4799 Some(c) if allowed.contains(&c) => Ok(()),
4800 other => bail!("vote {other:?} is not one of {allowed:?}"),
4801 },
4802 )
4803 .await;
4804 if self.state.resource_block.is_some() {
4805 self.stop_for_resource()?;
4806 return Ok(false);
4807 }
4808 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4809 let agent_id = seat.agent.clone();
4810 self.state.seats.insert(seat.key.clone(), seat);
4811 match res {
4812 Ok((v, _)) => {
4813 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4814 rec.vote = v.label();
4815 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4816 } else {
4817 self.state.votes.push(VoteRecord {
4818 judge: j + 1,
4819 agent: agent_id,
4820 vote: v.label(),
4821 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4822 changed: false,
4823 });
4824 }
4825 self.state.event(
4826 "recover",
4827 format!("judge {} voted again after the limit", j + 1),
4828 );
4829 }
4830 Err(e) => {
4831 self.state
4832 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4833 }
4834 }
4835 }
4836
4837 let recovered_keys: BTreeSet<String> = recovered
4841 .iter()
4842 .map(|&j| format!("judge-{}", j + 1))
4843 .collect();
4844 self.state
4845 .quota
4846 .retain(|q| !recovered_keys.contains(&q.seat));
4847 for loss in judge_losses.into_iter().chain(vote_losses) {
4851 if recovered_keys.contains(&loss.seat) {
4852 continue;
4853 }
4854 self.state.quota.retain(|q| q.seat != loss.seat);
4855 self.state.quota.push(loss);
4856 }
4857
4858 self.state.tally = None;
4860 self.tally()?;
4861 Ok(self
4862 .state
4863 .tally
4864 .as_ref()
4865 .map(|t| t.met_quorum)
4866 .unwrap_or(false))
4867 }
4868
4869 async fn graft(&mut self) -> Result<()> {
4885 let Err(e) = self.graft_inner().await else {
4886 return Ok(());
4887 };
4888 if e.is::<crate::repo_lock::ResourceBlocked>() {
4889 return Err(e);
4890 }
4891 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4892 self.state.status = RunStatus::Blocked;
4893 self.state.event("graft", format!("blocked: {e:#}"));
4894 self.state.save()?;
4895 return Err(e);
4896 }
4897 if let Some(rec) = self.state.graft.clone()
4902 && !rec.outcome.settled()
4903 {
4904 if let Some(winner) = self.state.winner().cloned() {
4905 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4906 if re.is::<crate::repo_lock::ResourceBlocked>() {
4907 return Err(re);
4908 }
4909 self.state.status = RunStatus::Blocked;
4910 self.state.event("graft", format!("blocked: {re:#}"));
4911 self.state.save()?;
4912 return Err(re);
4913 }
4914 }
4915 if let Some(g) = self.state.graft.as_mut() {
4916 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4917 }
4918 }
4919 self.state.event("graft", format!("skipped: {e:#}"));
4920 if let Err(e) = self.state.save() {
4921 tracing::warn!("save after graft: {e:#}");
4922 }
4923 Ok(())
4924 }
4925
4926 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4929 let _coordination = self.coordinate(wt).await?;
4930 crate::recovery::restore_repair(wt, pre)
4931 .await
4932 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4933 }
4934
4935 fn graft_skipped(&mut self, why: &str) {
4937 let message = format!("skipped: {why}");
4938 if !self
4939 .state
4940 .events
4941 .iter()
4942 .any(|e| e.node == "graft" && e.message == message)
4943 {
4944 self.state.event("graft", message);
4945 }
4946 }
4947
4948 async fn graft_inner(&mut self) -> Result<()> {
4949 if let Some(rec) = self.state.graft.clone()
4952 && !rec.outcome.settled()
4953 {
4954 if self.state.reviews.is_empty()
4958 && let Some(winner) = self.state.winner().cloned()
4959 {
4960 self.graft_restore(&winner.worktree, &rec.pre).await?;
4961 }
4962 if rec.outcome == GraftOutcome::WaitingForRestore {
4963 self.state.graft = None;
4964 self.state.save()?;
4965 } else {
4966 if let Some(g) = self.state.graft.as_mut() {
4967 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4968 }
4969 self.state.event(
4970 "graft",
4971 "rolled back: the previous round was interrupted; the winner is untouched",
4972 );
4973 self.state.save()?;
4974 return Ok(());
4975 }
4976 }
4977 let plan = match graft::plan(&self.state) {
4978 Ok(plan) => plan,
4979 Err(skip) => {
4980 if skip != graft::Skip::AlreadyDone {
4981 self.graft_skipped(skip.reason());
4982 self.state.save()?;
4983 }
4984 return Ok(());
4985 }
4986 };
4987 let Some(winner) = self.state.winner().cloned() else {
4988 return Ok(());
4989 };
4990 let wt = winner.worktree.clone();
4991 if !git::is_clean(&wt).await? {
4992 self.graft_skipped("the winner's worktree has uncommitted changes");
4993 self.state.save()?;
4994 return Ok(());
4995 }
4996 let pre = git::rev_parse(&wt, "HEAD").await?;
4997 self.state.graft = Some(GraftRecord {
5000 pre: pre.clone(),
5001 post: None,
5002 lines: None,
5003 limit: None,
5004 agent: None,
5005 outcome: GraftOutcome::Running,
5006 requested: graft::requested(&plan),
5007 adoption: None,
5008 adoption_error: None,
5009 });
5010 self.state.event(
5011 "graft",
5012 format!(
5013 "porting strengths of candidate(s) {} onto {}",
5014 plan.strengths
5015 .iter()
5016 .map(|(l, _, _)| l.to_string())
5017 .collect::<Vec<_>>()
5018 .join(", "),
5019 plan.winner
5020 ),
5021 );
5022 self.state.save()?;
5023
5024 let fix_prompt = prompt::graft(
5025 &self.state.instruction,
5026 plan.winner,
5027 &plan.strengths,
5028 &self.state.config.graph.language,
5029 );
5030 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5031 let sessions = self.state.config.graph.sessions;
5032 let artifacts = agent::artifacts_dir(&self.state.dir());
5033 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5034 let (_, seat, out) = self
5035 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5036 SeatJob {
5037 prompt: fix_prompt.clone(),
5038 spec,
5039 seat,
5040 cwd: wt.clone(),
5041 timeout,
5042 allow_write: true,
5043 sessions,
5044 artifacts: artifacts.clone(),
5045 stem: "graft".to_owned(),
5046 handover: None,
5047 }
5048 })
5049 .await;
5050 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5051 self.state.graft.as_mut().expect("graft record").outcome =
5054 GraftOutcome::WaitingForRestore;
5055 return self.stop_for_resource();
5056 }
5057 let mut why: Option<String> = None;
5058 match &out {
5059 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5060 why = Some("the fixer produced nothing usable".to_owned());
5061 }
5062 AgentOutcome::Ok(_) => {}
5063 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5064 AgentOutcome::Quota(o) => {
5065 self.state.quota.push(QuotaLoss::from_report(
5066 seat.key.clone(),
5067 "graft",
5068 o.quota.as_ref(),
5069 ));
5070 why = Some("rate limited (quota)".to_owned());
5071 }
5072 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5073 why = Some(format!(
5074 "the fixer failed: {}",
5075 e.lines()
5076 .next()
5077 .unwrap_or("")
5078 .chars()
5079 .take(160)
5080 .collect::<String>()
5081 ));
5082 }
5083 }
5084 let agent_id = seat.agent.clone();
5085 self.state.seats.insert(seat.key.clone(), seat);
5086
5087 let mut post = None;
5088 let mut lines = None;
5089 let _coordination = self.coordinate(&wt).await?;
5090 if why.is_none() {
5091 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5092 if let Ok(r) = &rescued {
5093 self.state.note_withheld("graft", &r.withheld);
5094 }
5095 let head = git::rev_parse(&wt, "HEAD").await?;
5096 let dirty = match &rescued {
5100 Ok(r) => git::status_porcelain(&wt)
5101 .await
5102 .map(|st| {
5103 st.lines().any(|l| {
5104 !r.withheld
5105 .iter()
5106 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5107 })
5108 })
5109 .unwrap_or(true),
5110 Err(_) => true,
5111 };
5112 if dirty {
5113 why = Some("the fixer left changes that could not be committed".to_owned());
5114 } else if head == pre {
5115 why = Some("the fixer changed nothing".to_owned());
5116 } else {
5117 lines = git::changed_lines(&wt, &pre, &head).await;
5118 post = Some(head);
5119 }
5120 }
5121 let (adoption, adoption_error) = match &out {
5122 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5123 _ => (None, None),
5124 };
5125 let requested = graft::requested(&plan);
5126 let outcome = match why {
5127 None => {
5128 self.state.event(
5129 "graft",
5130 format!(
5131 "applied: {}",
5132 lines.map_or_else(
5133 || "changed lines unavailable".to_owned(),
5134 |n| format!("{n} changed lines")
5135 )
5136 ),
5137 );
5138 GraftOutcome::Applied
5139 }
5140 Some(why) => {
5141 crate::recovery::restore_repair(&wt, &pre)
5144 .await
5145 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5146 self.state.event(
5147 "graft",
5148 format!("rolled back: {why}; the winner is untouched"),
5149 );
5150 GraftOutcome::RolledBack(why)
5151 }
5152 };
5153 self.state.graft = Some(GraftRecord {
5154 pre,
5155 post,
5156 lines,
5157 limit: None,
5158 agent: Some(agent_id),
5159 outcome,
5160 requested,
5161 adoption,
5162 adoption_error,
5163 });
5164 self.state.save()?;
5165 Ok(())
5166 }
5167
5168 async fn fold_losers(&mut self) -> Result<()> {
5171 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5172 return Ok(());
5173 };
5174 if !self.state.candidate_preservation.is_empty() {
5175 bail!("candidate preservation incomplete; refusing cleanup");
5176 }
5177 let repo = self.state.repo.clone();
5178 let _coordination = self.coordinate(&repo).await?;
5179 let mut folded = Vec::new();
5180 for i in 0..self.state.candidates.len() {
5181 let c = &self.state.candidates[i];
5182 if c.label == winner || c.folded {
5183 continue;
5184 }
5185 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5186 remove_preserved_worktree(&repo, &wt).await?;
5187 if git::branch_exists(&repo, &branch).await?
5188 && !git::branch_delete(&repo, &branch).await?
5189 {
5190 bail!("candidate branch removal refused");
5191 }
5192 self.state.candidates[i].folded = true;
5193 self.state.save()?;
5194 folded.push(label.to_string());
5195 }
5196 let root = self.state.worktree_root();
5198 for j in 1..=self.roles.judges.len() {
5199 let wt = root.join(format!("judge-{j}"));
5200 if wt.exists() {
5201 remove_preserved_worktree(&repo, &wt).await?;
5202 }
5203 }
5204 if self.state.config.graph.advise {
5207 for k in 1..=self.state.config.graph.advisors {
5208 let wt = root.join(format!("advisor-{k}"));
5209 if wt.exists() {
5210 remove_preserved_worktree(&repo, &wt).await?;
5211 }
5212 }
5213 }
5214 if !folded.is_empty() {
5215 self.state
5216 .event("fold", format!("folded candidates {}", folded.join(", ")));
5217 self.state.save()?;
5218 }
5219 Ok(())
5220 }
5221
5222 async fn hold_base_sync(&mut self) -> Result<bool> {
5225 self.hold_base_sync_as(None).await
5226 }
5227
5228 async fn hold_base_sync_kind(
5229 &mut self,
5230 kind: crate::base_sync_recovery::StopKind,
5231 ) -> Result<()> {
5232 self.hold_base_sync_as(Some(kind)).await?;
5233 Ok(())
5234 }
5235
5236 async fn hold_base_sync_as(
5237 &mut self,
5238 kind: Option<crate::base_sync_recovery::StopKind>,
5239 ) -> Result<bool> {
5240 if !self.state.base_sync_blocked() {
5241 self.state.base_sync_recovery.stop = None;
5242 return Ok(false);
5243 }
5244 let head = match self.state.winner() {
5245 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5246 .await
5247 .unwrap_or_default(),
5248 None => String::new(),
5249 };
5250 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5251 && !authorization.consumed
5252 && authorization.stop.head != head
5253 {
5254 authorization.refuse(format!(
5255 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5256 authorization.stop.head
5257 ));
5258 }
5259 self.state.record_base_sync_stop(&head);
5260 if let Some(kind) = kind {
5261 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5262 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5263 stop.kind = kind;
5264 }
5265 }
5266 self.state.save()?;
5267 Ok(true)
5268 }
5269
5270 async fn sync_to_base(&mut self) -> Result<()> {
5312 if self.state.status == RunStatus::AlreadyInBase {
5313 return Ok(());
5314 }
5315 self.hold_base_sync().await?;
5316 let conflicted = self
5317 .state
5318 .base_sync
5319 .as_ref()
5320 .is_some_and(|s| s.conflict.is_some());
5321 let Some(winner) = self.state.winner().cloned() else {
5322 return Ok(());
5323 };
5324
5325 let repo = self.state.repo.clone();
5326 let remote = self.state.config.merge.remote.clone();
5327 let base_branch = self.state.base_branch.clone();
5328 let tracking = format!("{remote}/{base_branch}");
5329
5330 let fetched_base =
5331 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5332 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5337 return Ok(());
5338 };
5339
5340 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5341 self.state.ensure_base_episode();
5342 let mut pending = self
5343 .state
5344 .base_sync_recovery
5345 .operations
5346 .last()
5347 .filter(|o| !o.finished)
5348 .cloned();
5349 let scratch = self.state.dir().join("base-sync");
5350 if git::rebase_in_progress(&scratch).await {
5351 if pending.is_none()
5354 && self.state.base_sync_recovery.operations.is_empty()
5355 && !conflicted
5356 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5357 .await
5358 .ok()
5359 .as_deref()
5360 == Some(head.as_str())
5361 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5362 && git::is_clean(&winner.worktree).await?
5363 {
5364 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5365 if self.state.base_sync.is_none() {
5366 self.state.base_sync = Some(BaseSync {
5367 tip: saved_tip.clone(),
5368 behind: 0,
5369 attempts: 0,
5370 conflict: None,
5371 already_in: None,
5372 });
5373 }
5374 self.state.reserve_base_operation(&head, &saved_tip);
5375 self.state.event(
5376 "base-recovery",
5377 "imported standing legacy rebase as a spent synchronization reservation",
5378 );
5379 self.state.save()?;
5380 pending = self.state.base_sync_recovery.operations.last().cloned();
5381 }
5382 let expected = pending.as_ref().map(|op| op.tip.as_str());
5383 if expected.is_none()
5384 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5385 {
5386 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5387 if self.state.base_sync.is_none() {
5388 self.state.base_sync = Some(BaseSync {
5389 tip: tip.clone(),
5390 behind: 0,
5391 attempts: 0,
5392 conflict: None,
5393 already_in: None,
5394 });
5395 }
5396 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5397 self.state.event("land", why);
5398 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5399 .await?;
5400 return Ok(());
5401 }
5402 }
5403 if let Some(op) = &pending {
5404 let mut replay_proven =
5406 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5407 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5408 let wait = repository_wait(&self.state, None);
5409 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5410 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5411 Ok(true) => {
5412 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5413 self.state.save()?;
5414 git::worktree_remove(&repo, &scratch).await.ok();
5415 replay_proven = true;
5416 }
5417 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5418 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5419 }
5420 }
5421 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5422 if self.state.foreign_checkout.is_some() {
5423 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5424 .await
5425 .ok()
5426 .as_deref()
5427 == Some(op.head.as_str())
5428 && git::rev_parse(&repo, &format!("{head}^2"))
5429 .await
5430 .ok()
5431 .as_deref()
5432 == Some(op.tip.as_str());
5433 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5434 .await
5435 .is_ok_and(|m| m.is_empty())
5436 && let Ok(added) =
5437 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5438 {
5439 replay_proven = true;
5440 for commit in added {
5441 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5442 replay_proven = false;
5443 break;
5444 }
5445 }
5446 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5447 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5448 }
5449 }
5450 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5454 || (head != op.head
5455 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5456 .await?
5457 .is_empty())
5458 || (!git::is_clean(&winner.worktree).await?
5459 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5460 .await
5461 .is_ok_and(|o| o.ok()))
5462 || !replay_proven
5463 {
5464 let why = format!(
5465 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5466 op.id
5467 );
5468 let sync = self
5469 .state
5470 .base_sync
5471 .as_mut()
5472 .context("reserved synchronization has no lifetime accounting")?;
5473 sync.conflict = Some(why.clone());
5474 self.state.event("land", why);
5475 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5476 .await?;
5477 return Ok(());
5478 }
5479 if head != op.head {
5480 git::sync_to_head(&winner.worktree).await?;
5481 }
5482 tip = op.tip.clone();
5483 }
5484 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5485 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5486 if pending.is_none()
5487 && !conflicted
5488 && fetched_base
5489 && self
5490 .state
5491 .base_sync_recovery
5492 .episodes
5493 .last()
5494 .is_some_and(|e| e.boundary.is_some())
5495 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5496 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5497 && owner.runs.last() == Some(&self.state.id)
5498 && owner.status != crate::queue::TaskStatus::Done
5499 && !owner.fresh_start
5500 && owner.review_branch.is_none()
5501 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5502 && let Some(approval) = owner.resumption.as_ref()
5503 && approval.instruction == owner.instruction
5504 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5505 && git::is_clean(&winner.worktree).await?
5506 && self.state.renew_base_episode(&task, approval, &head, &tip)
5507 {
5508 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5509 self.state.save()?;
5510 }
5511
5512 if (behind > 0 || conflicted || head == tip)
5521 && self
5522 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5523 .await?
5524 {
5525 return Ok(());
5526 }
5527 if conflicted && self.state.base_recovery_pending() {
5528 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5529 let owner =
5530 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5531 let owner_valid = owner.as_ref().is_ok_and(|t| {
5532 t.runs.last() == Some(&self.state.id)
5533 && t.status != crate::queue::TaskStatus::Done
5534 && !t.fresh_start
5535 && t.review_branch.is_none()
5536 });
5537 let invalidation = if !owner_valid {
5538 Some("latest task ownership changed")
5539 } else if !owner.as_ref().is_ok_and(|t| {
5540 authorization
5541 .generation
5542 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5543 }) {
5544 Some("owner approval generation changed")
5545 } else if let Some(scope) = &authorization.scope {
5546 if scope.tip != tip {
5547 Some("fetched base changed")
5548 } else if !self
5549 .state
5550 .base_sync_recovery
5551 .episodes
5552 .last()
5553 .is_some_and(|e| e.id == scope.episode)
5554 {
5555 Some("active synchronization interval changed")
5556 } else if !owner
5557 .as_ref()
5558 .is_ok_and(|t| t.instruction == scope.instruction)
5559 {
5560 Some("task instruction changed")
5561 } else {
5562 None
5563 }
5564 } else {
5565 None
5566 };
5567 if let Some(reason) = invalidation {
5568 let authorization = self
5569 .state
5570 .base_sync_recovery
5571 .authorizations
5572 .last_mut()
5573 .unwrap();
5574 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5575 self.state.save()?;
5576 return Ok(());
5577 }
5578 if !fetched_base
5579 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5580 || !git::is_clean(&winner.worktree).await?
5581 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5582 {
5583 return Ok(());
5584 }
5585 }
5586 let recovery = conflicted && self.state.consume_base_recovery(&head);
5587 if conflicted {
5588 self.state.save()?;
5590 if !recovery {
5591 return Ok(());
5592 }
5593 }
5594
5595 if behind == 0 {
5596 if let Some(from) = self
5605 .state
5606 .rebase_fixes
5607 .iter()
5608 .rev()
5609 .find_map(|r| r.from.clone())
5610 && from != head
5611 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5612 .await
5613 .is_ok_and(|o| o.ok())
5614 {
5615 git::sync_to_head(&winner.worktree).await?;
5616 }
5617 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5623 if let Some(why) = &conflict {
5624 self.state.status = RunStatus::Blocked;
5625 self.state.event("land", why.clone());
5626 }
5627 self.state.base_sync = Some(BaseSync {
5628 tip,
5629 behind: 0,
5630 attempts,
5631 conflict,
5632 already_in: None,
5633 });
5634 if pending.is_some() {
5635 self.state.finish_base_operation(Some(&head));
5636 }
5637 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5638 .await?;
5639 self.state.save()?;
5640 return Ok(());
5641 }
5642
5643 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5644 let why = format!(
5645 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5646 synchronization(s); synchronizing again would only race it",
5647 winner.branch
5648 );
5649 self.state.status = RunStatus::Blocked;
5650 self.state.base_sync = Some(BaseSync {
5651 tip,
5652 behind,
5653 attempts,
5654 conflict: Some(why.clone()),
5655 already_in: None,
5656 });
5657 self.state.event("land", why);
5658 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5659 .await?;
5660 self.state.save()?;
5661 return Ok(());
5662 }
5663
5664 if self.state.foreign_checkout.is_some() {
5665 let wait = repository_wait(&self.state, None);
5668 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5669 if self.state.base_sync.is_none() {
5670 self.state.base_sync = Some(BaseSync {
5671 tip: tip.clone(),
5672 behind,
5673 attempts: 0,
5674 conflict: None,
5675 already_in: None,
5676 });
5677 }
5678 self.state.reserve_base_operation(&head, &tip);
5679 self.state.save()?;
5680 let merged = git::git_raw(
5681 &winner.worktree,
5682 &[
5683 "-c",
5684 "core.hooksPath=/dev/null",
5685 "merge",
5686 "--no-ff",
5687 "--no-autostash",
5688 "--no-edit",
5689 "-m",
5690 &format!("Merge {tracking} before verifying"),
5691 &tip,
5692 ],
5693 )
5694 .await?;
5695 let conflict = if merged.ok() {
5696 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5697 self.state.event(
5698 "land",
5699 format!("merged {tracking} into {} before verifying", winner.branch),
5700 );
5701 None
5702 } else {
5703 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5704 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5705 }
5706 let why = format!(
5707 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5708 winner.branch,
5709 merged.stderr.chars().take(600).collect::<String>()
5710 );
5711 self.state.status = RunStatus::Blocked;
5712 self.state.event("land", &why);
5713 Some(why)
5714 };
5715 let synced = conflict.is_none();
5716 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5717 self.state
5718 .finish_base_operation(synced.then_some(fresh.as_str()));
5719 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5720 self.state.base_sync = Some(BaseSync {
5721 tip,
5722 behind: if synced { 0 } else { behind },
5723 attempts,
5724 conflict,
5725 already_in: None,
5726 });
5727 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5728 .await?;
5729 self.state.save()?;
5730 return Ok(());
5731 }
5732
5733 self.state.event(
5734 "land",
5735 format!(
5736 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5737 winner.branch
5738 ),
5739 );
5740 self.state.save()?;
5741
5742 let branch_tracking = format!("{remote}/{}", winner.branch);
5748 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5749 let remote_tip = if self.state.task_pr_branch().is_none()
5750 && matches!(&fetched_branch, Ok(o) if o.ok())
5751 {
5752 git::rev_parse(&repo, &branch_tracking).await.ok()
5753 } else {
5754 None
5755 };
5756 if let Some(theirs) = &remote_tip
5760 && !git::is_ancestor(&repo, theirs, &head).await
5761 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5762 .await
5763 .is_ok_and(|missing| missing.is_empty())
5764 {
5765 let why = format!(
5766 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5767 them",
5768 short(theirs),
5769 winner.branch
5770 );
5771 self.state.status = RunStatus::Blocked;
5772 self.state.base_sync = Some(BaseSync {
5773 tip,
5774 behind,
5775 attempts,
5776 conflict: Some(why.clone()),
5777 already_in: None,
5778 });
5779 self.state.event("land", why);
5780 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5781 .await?;
5782 self.state.save()?;
5783 return Ok(());
5784 }
5785
5786 if self.state.base_sync.is_none() {
5787 self.state.base_sync = Some(BaseSync {
5788 tip: tip.clone(),
5789 behind,
5790 attempts: 0,
5791 conflict: None,
5792 already_in: None,
5793 });
5794 }
5795 self.state.reserve_base_operation(&head, &tip);
5796 self.state.save()?;
5797 let scratch = self.state.dir().join("base-sync");
5798 let rebased =
5799 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5800 .await
5801 {
5802 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5803 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5804 Err(e) => Err(e),
5805 };
5806 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5807 let stop_kind = match &rebased {
5808 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5809 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5810 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5811 };
5812 match rebased {
5813 Ok(None) => {
5814 git::sync_to_head(&winner.worktree).await?;
5818 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5819 self.state.finish_base_operation(Some(&fresh));
5820 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5821 let mut conflict = None;
5822 if let Some(pinned) = &remote_tip {
5823 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5824 match pushed {
5825 Ok(o) if o.ok() => self.state.event(
5826 "land",
5827 format!("pushed rebased {} to {remote}", winner.branch),
5828 ),
5829 Ok(o) => {
5830 conflict = Some(format!(
5831 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5832 winner.branch,
5833 short(pinned),
5834 o.stderr.chars().take(600).collect::<String>()
5835 ));
5836 }
5837 Err(e) => {
5838 conflict = Some(format!(
5839 "rebased {} locally but could not push it: {e:#}",
5840 winner.branch
5841 ));
5842 }
5843 }
5844 }
5845 if let Some(why) = &conflict {
5846 self.state.status = RunStatus::Blocked;
5847 self.state.event("land", why.clone());
5848 }
5849 self.state.base_sync = Some(BaseSync {
5850 tip: tip.clone(),
5851 behind: 0,
5852 attempts,
5853 conflict,
5854 already_in: None,
5855 });
5856 self.state
5857 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5858 }
5859 Ok(Some(conflict)) => {
5860 self.state.finish_base_operation(None);
5861 let why = format!(
5862 "{} conflicts with {tracking} and did not rebase: {}",
5863 winner.branch,
5864 conflict.chars().take(600).collect::<String>()
5865 );
5866 self.state.status = RunStatus::Blocked;
5867 self.state.base_sync = Some(BaseSync {
5868 tip,
5869 behind,
5870 attempts,
5871 conflict: Some(why.clone()),
5872 already_in: None,
5873 });
5874 self.state.event("land", why);
5875 }
5876 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883 Err(e) => {
5884 self.state.finish_base_operation(None);
5885 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5886 self.state.status = RunStatus::Blocked;
5887 self.state.base_sync = Some(BaseSync {
5888 tip,
5889 behind,
5890 attempts,
5891 conflict: Some(why.clone()),
5892 already_in: None,
5893 });
5894 self.state.event("land", why);
5895 }
5896 }
5897 if !self.state.base_sync_blocked() {
5898 self.state.base_sync_recovery.stop = None;
5899 }
5900 self.hold_base_sync_kind(stop_kind).await?;
5901 self.state.save()?;
5902 Ok(())
5903 }
5904
5905 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5911 if self.state.task_pr_branch().is_some() {
5914 return None;
5915 }
5916 let repo = self.state.repo.clone();
5917 let remote = self.state.config.merge.remote.clone();
5918 let fetched = git::fetch(&repo, &remote, branch).await;
5919 if !matches!(&fetched, Ok(o) if o.ok()) {
5920 return None;
5921 }
5922 let branch_tracking = format!("{remote}/{branch}");
5923 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5924 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5925 return None;
5926 }
5927 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5928 .await
5929 .is_ok_and(|missing| missing.is_empty())
5930 {
5931 return Some(format!(
5932 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5933 them",
5934 short(&theirs)
5935 ));
5936 }
5937 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5938 Ok(o) if o.ok() => {
5939 self.state
5940 .event("land", format!("pushed rebased {branch} to {remote}"));
5941 None
5942 }
5943 Ok(o) => Some(format!(
5944 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5945 someone may have pushed): {}",
5946 short(&theirs),
5947 o.stderr.chars().take(600).collect::<String>()
5948 )),
5949 Err(e) => Some(format!(
5950 "{branch} is rebased locally but could not be pushed: {e:#}"
5951 )),
5952 }
5953 }
5954
5955 async fn settle_already_in(
5972 &mut self,
5973 branch: &str,
5974 tip: &str,
5975 head: &str,
5976 attempts: usize,
5977 behind: usize,
5978 ) -> Result<bool> {
5979 let repo = self.state.repo.clone();
5980 let remote = self.state.config.merge.remote.clone();
5981 let start = self.state.base_commit.clone();
5982 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5983 Ok(Some(e)) => e,
5984 Ok(None) => return Ok(false),
5985 Err(e) => {
5986 tracing::warn!("already-in-base check for {branch}: {e:#}");
5987 return Ok(false);
5988 }
5989 };
5990 let mut verified = vec![head.to_owned()];
5991 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5992 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5993 if matches!(&fetched, Ok(o) if o.ok())
5994 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5995 && theirs != head
5996 {
5997 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5998 Ok(Some(_)) => verified.push(theirs),
5999 _ => return Ok(false),
6000 }
6001 }
6002 let base_branch = self.state.base_branch.clone();
6003 let message = format!(
6004 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6005 land",
6006 evidence.names(),
6007 evidence.proof.as_str()
6008 );
6009 let closed =
6010 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6011 .await;
6012 match closed {
6013 Ok(Ok(url)) => self
6014 .state
6015 .event("land", format!("closed {url}: superseded on {base_branch}")),
6016 Ok(Err(why)) => self
6017 .state
6018 .event("land", format!("did not close a pull request: {why}")),
6019 Err(e) => {
6020 let why = format!(
6021 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6022 not be closed ({e:#}); resume to retry"
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip: tip.to_owned(),
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6034 .await?;
6035 self.state.save()?;
6036 return Ok(true);
6037 }
6038 }
6039 self.state.status = RunStatus::AlreadyInBase;
6040 self.state.base_sync_recovery.stop = None;
6041 self.state.base_sync = Some(BaseSync {
6042 tip: tip.to_owned(),
6043 behind,
6044 attempts,
6045 conflict: None,
6046 already_in: Some(evidence),
6047 });
6048 self.state.event("land", message);
6049 self.state.save()?;
6050 self.settle_questions();
6051 Ok(true)
6052 }
6053
6054 fn landing_base(&self) -> String {
6064 self.state
6065 .base_sync
6066 .as_ref()
6067 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6068 }
6069
6070 pub async fn fix_selected(
6103 &mut self,
6104 ids: &[String],
6105 reason: &str,
6106 allow_stale: bool,
6107 ) -> Result<()> {
6108 let reason = reason.trim();
6109 if reason.is_empty() {
6110 bail!("a fix request needs a reason — that is the operator's own record of why");
6111 }
6112 if ids.is_empty() {
6113 bail!("no finding id given");
6114 }
6115 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6116 bail!(
6117 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6118 has already concluded — can be given a targeted fix. A run still \
6119 in progress should simply be resumed; a `merged` run's branch has \
6120 already landed, so its answer is a fresh `magi review <branch>`, \
6121 not reopening this run's own record",
6122 self.state.id,
6123 self.state.status.as_str()
6124 );
6125 }
6126 let Some(winner) = self.state.winner().cloned() else {
6127 bail!("run {} has no winning candidate to fix", self.state.id);
6128 };
6129 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6130 bail!(
6131 "branch `{}` no longer exists; this run cannot be extended",
6132 winner.branch
6133 );
6134 }
6135 let home = crate::run::home();
6136 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6137 bail!(
6138 "run {} is currently being worked on by another magi process",
6139 self.state.id
6140 );
6141 }
6142 let _claim = FixClaim::acquire(&self.state.dir())?;
6148
6149 let mut seen = BTreeSet::new();
6153 let mut findings = Vec::new();
6154 let mut missing = Vec::new();
6155 for id in ids {
6156 if !seen.insert(id.clone()) {
6157 continue;
6158 }
6159 match self.state.finding(id) {
6160 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6161 id: f.id.clone(),
6162 severity: f.severity,
6163 reviewer_vote: rec.vote,
6164 round: round.round,
6165 round_head: round.head.clone(),
6166 reviewer: rec.reviewer,
6167 agent: rec.agent.clone(),
6168 file: f.file.clone(),
6169 line: f.line,
6170 title: f.title.clone(),
6171 detail: f.detail.clone(),
6172 outcome: OperatorFixOutcome::Pending,
6173 }),
6174 None => missing.push(id.clone()),
6175 }
6176 }
6177 if !missing.is_empty() {
6178 bail!(
6179 "unknown finding id(s): {}; nothing was changed",
6180 missing.join(", ")
6181 );
6182 }
6183
6184 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6185 let stale_details: Vec<(String, String)> = findings
6186 .iter()
6187 .filter(|f| f.round_head != head_at_request)
6188 .map(|f| (f.id.clone(), f.round_head.clone()))
6189 .collect();
6190 let stale = !stale_details.is_empty();
6191 if stale && !allow_stale {
6192 bail!(
6193 "the branch has moved since some finding(s) were raised — {} — now \
6194 at {}; pass --allow-stale to fix anyway, or re-run review first",
6195 stale_details
6196 .iter()
6197 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6198 .collect::<Vec<_>>()
6199 .join(", "),
6200 short(&head_at_request)
6201 );
6202 }
6203
6204 let request = OperatorFixRequest {
6205 requested_at: Timestamp::now(),
6206 reason: reason.to_owned(),
6207 findings,
6208 head_at_request: head_at_request.clone(),
6209 allow_stale,
6210 stale,
6211 fix: None,
6212 result_head: None,
6213 follow_up_review_run: None,
6214 };
6215 self.state.event(
6216 "fix",
6217 format!(
6218 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6219 request.findings.len(),
6220 request
6221 .findings
6222 .iter()
6223 .map(|f| f.id.as_str())
6224 .collect::<Vec<_>>()
6225 .join(", "),
6226 ),
6227 );
6228 self.state.operator_fixes.push(request);
6235 self.state.save()?;
6236 let request_index = self.state.operator_fixes.len() - 1;
6237
6238 let repo = self.state.repo.clone();
6247 let coordination = self.coordinate(&repo).await?;
6248 if winner.worktree.exists() {
6249 let dirty = git::git(
6252 &winner.worktree,
6253 &["status", "--porcelain", "--untracked-files=all"],
6254 )
6255 .await?;
6256 let only_withheld = dirty.lines().all(|l| {
6257 l.strip_prefix("?? ")
6258 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6259 });
6260 if !only_withheld {
6261 bail!(
6262 "`{}` has uncommitted changes; refusing to touch it — commit or \
6263 discard them first",
6264 winner.worktree.display()
6265 );
6266 }
6267 git::worktree_remove(&self.state.repo, &winner.worktree)
6268 .await
6269 .ok();
6270 }
6271 let fix_worktree = self.state.worktree_root().join("operator-fix");
6272 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6273 git::git(
6274 &self.state.repo,
6275 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6276 )
6277 .await
6278 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6279 if !git::is_clean(&fix_worktree).await? {
6280 git::worktree_remove(&self.state.repo, &fix_worktree)
6281 .await
6282 .ok();
6283 bail!(
6284 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6285 winner.branch
6286 );
6287 }
6288 let repo_for_setup = self.state.repo.clone();
6289 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6290 Ok(()) => {
6291 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6292 }
6293 Err(e) => Err(e),
6294 };
6295 if let Err(e) = setup {
6296 git::worktree_remove(&self.state.repo, &fix_worktree)
6297 .await
6298 .ok();
6299 return Err(e);
6300 }
6301
6302 self.state.resource_block = None;
6305
6306 let run_id = self.state.id.clone();
6307 let prompts = self.state.config.prompts.clone();
6308 let language = self.state.config.graph.language.clone();
6309 let sessions = self.state.config.graph.sessions;
6310 let artifacts = agent::artifacts_dir(&self.state.dir());
6311 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6312 .findings
6313 .iter()
6314 .map(|f| Finding {
6315 id: f.id.clone(),
6316 severity: f.severity,
6317 file: f.file.clone(),
6318 line: f.line,
6319 title: f.title.clone(),
6320 detail: f.detail.clone(),
6321 })
6322 .collect();
6323 let fix_prompt = prompt::operator_fix_in(
6324 &self.state.instruction,
6325 &finding_list,
6326 reason,
6327 &stale_details,
6328 &head_at_request,
6329 &language,
6330 &self.state.config.graph.github_language,
6331 );
6332 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6333 drop(coordination);
6334 let (job, seat, out) = self
6335 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6336 prompt: fix_prompt.clone(),
6337 spec,
6338 seat,
6339 cwd: fix_worktree.clone(),
6340 timeout,
6341 allow_write: true,
6342 sessions,
6343 artifacts: artifacts.clone(),
6344 stem: "operator-fix".to_owned(),
6345 handover: None,
6346 })
6347 .await;
6348 let agent_id = seat.agent.clone();
6349
6350 let mut fix = FixRecord {
6351 agent: agent_id,
6352 addressed: Vec::new(),
6353 rejected: Vec::new(),
6354 notes: String::new(),
6355 committed: false,
6356 failed: None,
6357 duration_ms: 0,
6358 continuation: None,
6359 };
6360 let mut final_seat = seat.clone();
6361 match out {
6362 AgentOutcome::Ok(o) => {
6363 fix.duration_ms = o.duration_ms;
6364 let parsed = verdict::extract_json::<FixReport>(&o.text);
6365 let incomplete_reason = match &parsed {
6366 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6367 "the reply parsed, but it reported a command whose own CLI \
6368 never confirmed an exit status"
6369 .to_owned(),
6370 ),
6371 Ok(_) => None,
6372 Err(e) => Some(e.to_string()),
6373 };
6374 match incomplete_reason {
6375 None => {
6376 let report = parsed.expect("checked Ok above");
6377 fix.addressed = report.addressed;
6378 fix.rejected = report.rejected;
6379 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6380 }
6381 Some(reason) => {
6382 let (resumed_seat, resolved, failure, cont) = self
6383 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6384 .await;
6385 fix.duration_ms += cont.cumulative_wait_ms;
6386 fix.continuation = Some(cont);
6387 final_seat = resumed_seat;
6388 match resolved {
6389 Some(report) => {
6390 fix.addressed = report.addressed;
6391 fix.rejected = report.rejected;
6392 fix.notes =
6393 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6394 }
6395 None => fix.failed = failure,
6396 }
6397 }
6398 }
6399 }
6400 AgentOutcome::Dropped(o) => {
6401 fix.duration_ms = o.duration_ms;
6402 let why = o
6403 .dropped
6404 .as_ref()
6405 .map(|d| d.why.as_str())
6406 .unwrap_or("the CLI ended the stream without delivering its answer");
6407 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6408 }
6409 AgentOutcome::Quota(o) => {
6410 self.state.quota.push(QuotaLoss::from_report(
6411 final_seat.key.clone(),
6412 "fix",
6413 o.quota.as_ref(),
6414 ));
6415 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6416 }
6417 AgentOutcome::Failed(e) => fix.failed = Some(e),
6418 AgentOutcome::ResourceBlocked(e) => {
6419 fix.failed = Some(format!("the fixer was not started: {e}"));
6420 }
6421 }
6422 if fix.continuation.is_none() {
6425 fix.continuation = Some(ContinuationRecord::not_needed());
6426 }
6427 self.state.seats.insert(final_seat.key.clone(), final_seat);
6428
6429 let rescue_message = format!(
6430 "magi: operator-selected fix ({}) (uncommitted work)",
6431 self.state.operator_fixes[request_index]
6432 .findings
6433 .iter()
6434 .map(|f| f.id.as_str())
6435 .collect::<Vec<_>>()
6436 .join(", ")
6437 );
6438 let repo = self.state.repo.clone();
6439 let coordination = self.coordinate(&repo).await?;
6440 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6441 self.state.note_withheld("fix", &r.withheld);
6442 }
6443 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6444 fix.committed = after != head_at_request;
6445 git::worktree_remove(&self.state.repo, &fix_worktree)
6446 .await
6447 .ok();
6448 drop(coordination);
6449
6450 self.state.event(
6451 "fix",
6452 match &fix.failed {
6453 Some(reason) => format!(
6454 "operator fix: adoption report was lost ({reason}); {}",
6455 if fix.committed {
6456 "committed"
6457 } else {
6458 "NO new commit"
6459 }
6460 ),
6461 None => format!(
6462 "operator fix: {} addressed, {} rejected, {}",
6463 fix.addressed.len(),
6464 fix.rejected.len(),
6465 if fix.committed {
6466 "committed"
6467 } else {
6468 "NO new commit"
6469 }
6470 ),
6471 },
6472 );
6473
6474 for f in &mut self.state.operator_fixes[request_index].findings {
6481 f.outcome = if fix.failed.is_some() {
6482 OperatorFixOutcome::Unreported
6483 } else if fix.addressed.contains(&f.id) {
6484 OperatorFixOutcome::Addressed
6485 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6486 OperatorFixOutcome::Rejected { why: r.why.clone() }
6487 } else {
6488 OperatorFixOutcome::Unreported
6489 };
6490 }
6491
6492 let committed = fix.committed;
6493 if committed {
6494 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6495 }
6496 self.state.operator_fixes[request_index].fix = Some(fix);
6497 self.state.save()?;
6500 if self.state.resource_block.is_some() {
6501 return self.stop_for_resource();
6502 }
6503
6504 if committed {
6505 self.state.event(
6506 "fix",
6507 format!(
6508 "operator fix committed {}; opening a follow-up review-only run",
6509 short(&after)
6510 ),
6511 );
6512 let origin =
6515 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6516 match Self::review(
6517 &self.state.repo,
6518 &winner.branch,
6519 self.state.config.clone(),
6520 origin,
6521 )
6522 .await
6523 {
6524 Ok(mut follow_up) => {
6525 follow_up.state.event(
6526 "start",
6527 format!(
6528 "requested by an operator fix on run {} for finding(s) {}",
6529 self.state.id,
6530 self.state.operator_fixes[request_index]
6531 .findings
6532 .iter()
6533 .map(|f| f.id.as_str())
6534 .collect::<Vec<_>>()
6535 .join(", "),
6536 ),
6537 );
6538 follow_up.state.save()?;
6539 let follow_up_id = follow_up.state.id.clone();
6540 let adopted = match crate::direct::adopt(
6543 &follow_up.state,
6544 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6545 ) {
6546 Ok(a) => a,
6547 Err(e) => {
6548 self.state.event(
6551 "fix",
6552 format!(
6553 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6554 ),
6555 );
6556 self.state.save()?;
6557 return Ok(());
6558 }
6559 };
6560 let executed = follow_up.execute().await;
6561 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6562 if let Some(a) = adopted {
6563 a.finish(&follow_up.state, executed);
6564 }
6565 if let Some(e) = failure {
6566 self.state.event(
6567 "fix",
6568 format!(
6569 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6570 ),
6571 );
6572 }
6573 self.state.operator_fixes[request_index].follow_up_review_run =
6574 Some(follow_up_id);
6575 }
6576 Err(e) => {
6577 self.state.event(
6578 "fix",
6579 format!("committed the fix but could not open a follow-up review: {e:#}"),
6580 );
6581 }
6582 }
6583 self.state.save()?;
6584 }
6585
6586 Ok(())
6587 }
6588
6589 async fn ask_fixer(
6609 &mut self,
6610 winner: &Candidate,
6611 node: &'static str,
6612 round: Option<usize>,
6613 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6614 ) -> (SeatJob, SeatState, AgentOutcome) {
6615 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6616 self.ask_fixer_among(attempts, node, round, None, build)
6617 .await
6618 }
6619
6620 async fn ask_fixer_among(
6625 &mut self,
6626 attempts: Vec<(AgentSpec, String)>,
6627 node: &'static str,
6628 round: Option<usize>,
6629 restore: Option<(&Path, &str)>,
6630 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6631 ) -> (SeatJob, SeatState, AgentOutcome) {
6632 let run_id = self.state.id.clone();
6633 let prompts = self.state.config.prompts.clone();
6634 let cache = self.state.config.cache_dir();
6635 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6636 let mut last = None;
6637 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6638 let seat = self.seat(&key, &spec.id);
6639 let mut job = build(spec.clone(), seat);
6640 if i > 0 {
6641 job.stem = format!("{}-{}", job.stem, spec.id);
6642 }
6643 let ctx = WaveCtx {
6644 carry_seats: false,
6645 pauses: Some((&self.pause, &self.interrupt)),
6646 run: &run_id,
6647 node,
6648 prompts: &prompts,
6649 cache: cache.as_deref(),
6650 round,
6651 };
6652 let (seat, out) =
6653 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6654 let advances = match &out {
6657 AgentOutcome::Ok(o) => agent::output_advances(o),
6658 AgentOutcome::ResourceBlocked(_) => false,
6659 _ => true,
6660 };
6661 if let Some(next) = ids.get(i + 1)
6662 && advances
6663 {
6664 self.state.seats.insert(seat.key.clone(), seat.clone());
6665 let class =
6666 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6667 let reason = match &out {
6668 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6669 other => fail_reason(other),
6670 };
6671 if let Some((wt, rev)) = restore
6675 && let Err(e) = self.graft_restore(wt, rev).await
6676 {
6677 let why =
6678 format!("could not restore the worktree before the next fixer: {e:#}");
6679 tracing::warn!("{why}");
6680 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6681 AgentOutcome::ResourceBlocked(why)
6682 } else {
6683 AgentOutcome::Failed(why)
6684 };
6685 last = Some((job, seat, out));
6686 break;
6687 }
6688 record_handover(
6689 &mut self.state,
6690 node,
6691 &seat.key,
6692 &spec.id,
6693 next,
6694 &class,
6695 &reason,
6696 );
6697 continue;
6698 }
6699 last = Some((job, seat, out));
6700 break;
6701 }
6702 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6703 if let AgentOutcome::Quota(output) = &out
6704 && node != "graft"
6705 {
6706 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6711 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6712 self.state.quota.push(loss);
6713 self.state.seats.insert(seat.key.clone(), seat.clone());
6714 self.state.status = RunStatus::Blocked;
6715 return (
6716 job,
6717 seat,
6718 AgentOutcome::ResourceBlocked(
6719 "fixer unavailable; restore agent availability and resume".into(),
6720 ),
6721 );
6722 }
6723 (job, seat, out)
6724 }
6725
6726 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6737 use crate::recovery::{Mode, Repair};
6738 if !self.state.reviews.is_empty() {
6739 return Ok(());
6740 }
6741 let Some(mut rec) = self.state.recovery.clone() else {
6742 return Ok(());
6743 };
6744 if rec.mode != Mode::VerifyRepair {
6745 return Ok(());
6746 }
6747 let wt = winner.worktree.clone();
6748 let mut head = git::rev_parse(&wt, "HEAD").await?;
6749 match rec.repair.clone() {
6750 Repair::Done { .. } => return Ok(()),
6751 Repair::WaitingForRestore { before } => {
6752 self.graft_restore(&wt, &before).await?;
6753 head = before;
6754 rec.repair = Repair::NotStarted;
6755 self.state.recovery = Some(rec.clone());
6756 self.state.save()?;
6757 }
6758 Repair::Started { before } => {
6759 if let Err(e) = self.graft_restore(&wt, &before).await {
6763 if e.is::<crate::repo_lock::ResourceBlocked>() {
6764 return Err(e);
6765 }
6766 return self
6767 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6768 .await;
6769 }
6770 rec.repair = Repair::Done {
6771 before: before.clone(),
6772 after: before,
6773 committed: false,
6774 failed: Some("interrupted; rolled back".to_owned()),
6775 };
6776 self.state
6777 .event("recovery", "the repair was interrupted and rolled back");
6778 self.state.recovery = Some(rec);
6779 self.state.save()?;
6780 return Ok(());
6781 }
6782 Repair::NotStarted => {}
6783 }
6784 if crate::recovery::recheck(&mut rec, &head) {
6785 self.state.event("recovery", rec.summary());
6786 self.state.recovery = Some(rec);
6787 self.state.save()?;
6788 return Ok(());
6789 }
6790 rec.repair = Repair::Started {
6791 before: head.clone(),
6792 };
6793 self.state.recovery = Some(rec.clone());
6794 self.state.save()?;
6795
6796 let language = self.state.config.graph.language.clone();
6797 let sessions = self.state.config.graph.sessions;
6798 let artifacts = agent::artifacts_dir(&self.state.dir());
6799 let fix_prompt = prompt::recovery_fix(
6800 &self.state.instruction,
6801 &rec,
6802 self.state.config.graph.review_rounds,
6803 &language,
6804 );
6805 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6806 self.state.event(
6807 "recovery",
6808 format!(
6809 "repair: the fixer starts from the failed verification on {}",
6810 short(&head)
6811 ),
6812 );
6813 let (_job, seat, out) = self
6814 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6815 prompt: fix_prompt.clone(),
6816 spec,
6817 seat,
6818 cwd: wt.clone(),
6819 timeout,
6820 allow_write: true,
6821 sessions,
6822 artifacts: artifacts.clone(),
6823 stem: "fix-recovery".to_owned(),
6824 handover: None,
6825 })
6826 .await;
6827 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6828 rec.repair = Repair::WaitingForRestore { before: head };
6829 self.state.recovery = Some(rec);
6830 return self.stop_for_resource();
6831 }
6832 let failed = match &out {
6833 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6834 AgentOutcome::Quota(o) => {
6835 self.state.quota.push(QuotaLoss::from_report(
6836 seat.key.clone(),
6837 "fix",
6838 o.quota.as_ref(),
6839 ));
6840 Some("rate limited (quota); the fixer could not run".to_owned())
6841 }
6842 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6843 AgentOutcome::Failed(e) => Some(e.clone()),
6844 };
6845 self.state.seats.insert(seat.key.clone(), seat);
6846 let coordination = self.coordinate(&wt).await?;
6847 let failed = match failed {
6848 Some(why) => {
6850 if let Err(e) = git::restore_to(&wt, &head).await {
6851 return self
6852 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6853 .await;
6854 }
6855 Some(why)
6856 }
6857 None => {
6858 if let Ok(r) = git::rescue_commit(
6859 &wt,
6860 "magi: recovery repair of the failed verification (uncommitted work)",
6861 )
6862 .await
6863 {
6864 self.state.note_withheld("fix", &r.withheld);
6865 }
6866 None
6867 }
6868 };
6869 let after = git::rev_parse(&wt, "HEAD").await?;
6870 drop(coordination);
6871 self.state.event(
6872 "recovery",
6873 match &failed {
6874 Some(why) => format!("repair: nothing kept ({why})"),
6875 None if after != head => {
6876 format!("repair: committed {}", short(&after))
6877 }
6878 None => "repair: the fixer made no commit".to_owned(),
6879 },
6880 );
6881 rec.repair = Repair::Done {
6882 before: head.clone(),
6883 committed: after != head,
6884 after,
6885 failed,
6886 };
6887 self.state.recovery = Some(rec);
6888 self.state.save()?;
6889 Ok(())
6890 }
6891
6892 async fn stop_recovery(
6895 &mut self,
6896 mut rec: crate::recovery::RecoveryRecord,
6897 head: &str,
6898 why: String,
6899 ) -> Result<()> {
6900 rec.repair = crate::recovery::Repair::Started {
6902 before: head.to_owned(),
6903 };
6904 self.state.recovery = Some(rec);
6905 self.state
6906 .event("recovery", format!("repair could not be undone: {why}"));
6907 self.state.status = RunStatus::Blocked;
6908 self.state.save()?;
6909 bail!("recovery repair could not be rolled back: {why}")
6910 }
6911
6912 async fn review_loop(&mut self) -> Result<()> {
6913 if self.hold_base_sync().await? {
6914 return Ok(());
6915 }
6916 loop {
6917 if self
6918 .state
6919 .base_sync
6920 .as_ref()
6921 .is_some_and(|s| s.conflict.is_some())
6922 {
6923 return Ok(());
6924 }
6925 if self.catchup_repairable() {
6926 let record = self.state.reviews.last().expect("repairable round").clone();
6927 let winner = self
6928 .state
6929 .winner()
6930 .cloned()
6931 .context("catch-up repair has no winner")?;
6932 if !self.fix_review_round(&winner, record).await? {
6933 return Ok(());
6934 }
6935 }
6936 self.review_loop_pass().await?;
6937 if !self.catchup_repairable() {
6938 return Ok(());
6939 }
6940 }
6941 }
6942
6943 fn catchup_repairable(&self) -> bool {
6946 self.state.reviews.len() < self.state.config.graph.review_rounds
6947 && self.state.reviews.last().is_some_and(|r| {
6948 !r.clean
6949 && r.blocking == 0
6950 && r.fix.is_none()
6951 && r.e2e_status() == E2eStatus::Failed
6952 })
6953 }
6954
6955 async fn review_loop_pass(&mut self) -> Result<()> {
6956 if self
6961 .state
6962 .base_sync
6963 .as_ref()
6964 .is_some_and(|s| s.conflict.is_some())
6965 {
6966 return Ok(());
6967 }
6968 let run_id = self.state.id.clone();
6973 let prompts = self.state.config.prompts.clone();
6974 let Some(winner) = self.state.winner().cloned() else {
6975 return Ok(());
6976 };
6977 let granted_review = self
6978 .state
6979 .repair_grants
6980 .last()
6981 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6982 .cloned();
6983 let fresh_review = granted_review
6984 .as_ref()
6985 .is_some_and(|g| self.state.reviews.len() == g.review_start);
6986 let max_rounds = granted_review
6987 .as_ref()
6988 .map_or(self.state.config.graph.review_rounds, |g| {
6989 g.review_start.saturating_add(1)
6990 });
6991 if let Some(status) =
7001 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7002 {
7003 if status == RunStatus::Gating {
7006 self.record_contested_handoff();
7007 }
7008 self.state.status = status;
7009 self.state.save()?;
7010 return Ok(());
7011 }
7012 self.state.status = RunStatus::Reviewing;
7013 if !fresh_review
7023 && self.state.reviews.last().is_some_and(|r| {
7024 r.e2e_status() == E2eStatus::ResourceBlocked
7025 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7026 })
7027 {
7028 let shell = self.state.config.shell();
7029 return self
7030 .stop_unresolved_review(
7031 "the last round's own verification never resolved",
7032 &shell,
7033 &winner.worktree,
7034 )
7035 .await;
7036 }
7037
7038 if granted_review.is_none() {
7039 self.recovery_repair(&winner).await?;
7040 }
7041 if self.resource_halt()? {
7042 return Ok(());
7043 }
7044
7045 let repo = self.state.repo.clone();
7046 let root = self.state.worktree_root();
7047 let language = self.state.config.graph.language.clone();
7048 let sessions = self.state.config.graph.sessions;
7049 let artifacts = self.merge_repair_artifacts();
7050 let base = self.landing_base();
7051 let base_short = short(&base);
7052 let reviewers = self.roles.reviewers.clone();
7053 let shell = self.state.config.shell();
7054
7055 for round in (self.state.reviews.len() + 1)..=max_rounds {
7056 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7057 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7058 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7059 let prev_verification = self
7068 .state
7069 .reviews
7070 .last()
7071 .and_then(|r| r.verification_summary(&head));
7072 let prior_section = match (&self.state.recovery, round) {
7075 (Some(rec), 1) if self.state.reviews.is_empty() => {
7076 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7077 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7078 Some(crate::recovery::head_of(
7079 &git::diff(&winner.worktree, reviewed, "HEAD")
7080 .await
7081 .unwrap_or_default(),
7082 crate::recovery::SINCE_DIFF_MAX,
7083 ))
7084 }
7085 _ => None,
7086 };
7087 Some(prompt::recovery_review(rec, since.as_deref()))
7088 }
7089 _ => None,
7090 };
7091
7092 let mut jobs = Vec::new();
7096 let coordination = self.coordinate(&repo).await?;
7097 for (r, spec) in reviewers.iter().cloned().enumerate() {
7098 let wt = root.join(format!("review-{}", r + 1));
7099 if wt.exists() {
7102 git::reset_detached(&wt, &head).await?;
7103 } else {
7104 git::worktree_add_detached(&repo, &wt, &head).await?;
7105 }
7106 self.setup_seat_worktree(&repo, &wt, false).await?;
7107 let seat_key = format!("review-{}", r + 1);
7108 let spec = pick_start_spec(
7111 &self.roles.reviewer_roster,
7112 spec,
7113 self.state.seat_history.get(&seat_key),
7114 );
7115 let seat = self.seat(&seat_key, &spec.id);
7116 jobs.push(SeatJob {
7117 prompt: prompt::review(&prompt::ReviewCtx {
7118 instruction: &self.state.instruction,
7119 branch: &winner.branch,
7120 base_short: &base_short,
7121 stat: &stat,
7122 patch: &patch,
7123 verification: prev_verification.as_ref(),
7124 prior: prior_section.as_deref(),
7125 reviewers: reviewers.len(),
7126 round,
7127 rounds: max_rounds,
7128 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7131 lens: Lens::for_seat(r),
7132 language: &language,
7133 github_language: &self.state.config.graph.github_language,
7134 }),
7135 spec,
7136 seat,
7137 cwd: wt,
7138 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7139 allow_write: false,
7140 sessions,
7141 artifacts: artifacts.clone(),
7142 stem: format!("review-{round}-{}", r + 1),
7143 handover: None,
7144 });
7145 }
7146
7147 drop(coordination);
7148 self.state.event(
7149 "review",
7150 format!(
7151 "round {round}: {} reviewers on {}",
7152 jobs.len(),
7153 short(&head)
7154 ),
7155 );
7156 let mut quota_losses = Vec::new();
7157 let review_retries = self.state.config.graph.retries;
7158 let review_cache = self.state.config.cache_dir();
7159 let ctx = WaveCtx {
7160 carry_seats: true,
7161 pauses: Some((&self.pause, &self.interrupt)),
7162 run: &run_id,
7163 node: "review",
7164 prompts: &prompts,
7165 cache: review_cache.as_deref(),
7166 round: Some(round),
7167 };
7168 let results = ask_json_wave::<Review>(
7169 jobs,
7170 Arc::clone(&self.sem),
7171 review_retries,
7172 &self.roles.reviewer_roster,
7173 &ctx,
7174 &mut quota_losses,
7175 &mut self.state,
7176 &|_: &Review| Ok(()),
7177 )
7178 .await;
7179 if self.state.resource_block.is_some() {
7180 return self.stop_for_resource();
7181 }
7182 let round_quota_missing = quota_losses.len();
7186 self.state.quota.extend(quota_losses);
7187
7188 let mut records = Vec::new();
7189 let mut all_findings = Vec::new();
7190 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7191 let agent_id = seat.agent.clone();
7192 self.state.seats.insert(seat.key.clone(), seat);
7193 let mut record = ReviewRecord {
7194 reviewer: r + 1,
7195 agent: agent_id,
7196 summary: String::new(),
7197 findings: Vec::new(),
7198 vote: None,
7199 failed: None,
7200 duration_ms: 0,
7201 attempts,
7207 };
7208 match res {
7209 Ok((review, out)) => {
7210 record.summary =
7218 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7219 record.vote = Some(review.vote);
7220 record.duration_ms = out.duration_ms;
7221 for (n, mut f) in review.findings.into_iter().enumerate() {
7222 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7225 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7226 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7227 f.file = f
7233 .file
7234 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7235 all_findings.push(f.clone());
7236 record.findings.push(f);
7237 }
7238 self.state.event(
7239 "review",
7240 format!(
7241 "round {round}: reviewer {} voted {} with {} finding(s)",
7242 r + 1,
7243 review.vote.label(),
7244 record.findings.len()
7245 ),
7246 );
7247 }
7248 Err(e) => {
7249 record.failed = Some(e.to_string());
7250 self.state.event(
7251 "review",
7252 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7253 );
7254 }
7255 }
7256 records.push(record);
7257 }
7258
7259 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7266 let vote_split =
7267 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7268 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7269 if vote_split {
7270 self.state.event(
7271 "review",
7272 format!(
7273 "round {round}: votes split ({}) — one round of reconsideration",
7274 initial_votes
7275 .iter()
7276 .map(|v| v.label())
7277 .collect::<Vec<_>>()
7278 .join(", ")
7279 ),
7280 );
7281 let panel: Vec<ReviewSeatReport<'_>> = records
7284 .iter()
7285 .filter_map(|r| {
7286 r.vote.map(|vote| ReviewSeatReport {
7287 reviewer: r.reviewer,
7288 vote,
7289 summary: &r.summary,
7290 findings: &r.findings,
7291 })
7292 })
7293 .collect();
7294
7295 let mut jobs = Vec::new();
7296 let mut seats_at = Vec::new();
7297 for (r, spec) in reviewers.iter().cloned().enumerate() {
7298 if records[r].vote.is_none() {
7302 continue;
7303 }
7304 let wt = root.join(format!("review-{}", r + 1));
7305 let seat_key = format!("review-{}", r + 1);
7306 let spec = self.occupant(&seat_key, spec);
7307 let seat = self.seat(&seat_key, &spec.id);
7308 let build = |with_patch: bool| {
7316 prompt::review_reconsider(&ReviewReconsiderCtx {
7317 instruction: &self.state.instruction,
7318 reviewer: r + 1,
7319 lens: Lens::for_seat(r),
7320 panel: &panel,
7321 patch: with_patch.then_some(ReviewPatch {
7322 branch: &winner.branch,
7323 base_short: &base_short,
7324 stat: &stat,
7325 patch: &patch,
7326 }),
7327 round,
7328 rounds: max_rounds,
7329 language: &language,
7330 })
7331 };
7332 let full = build(true);
7333 let prompt = if has_context(&spec, &seat, sessions) {
7334 build(false)
7335 } else {
7336 full.clone()
7337 };
7338 jobs.push(SeatJob {
7339 prompt,
7340 spec,
7341 seat,
7342 cwd: wt,
7343 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7344 allow_write: false,
7345 sessions,
7346 artifacts: artifacts.clone(),
7347 stem: format!("review-{round}-reconsider-{}", r + 1),
7348 handover: Some(full),
7349 });
7350 seats_at.push(r);
7351 }
7352
7353 let mut recon_quota_losses = Vec::new();
7354 let recon_cache = self.state.config.cache_dir();
7355 let recon_ctx = WaveCtx {
7356 carry_seats: true,
7357 pauses: Some((&self.pause, &self.interrupt)),
7358 run: &run_id,
7359 node: "review",
7360 prompts: &prompts,
7361 cache: recon_cache.as_deref(),
7362 round: Some(round),
7363 };
7364 let recon_results = ask_json_wave::<ReviewRevote>(
7365 jobs,
7366 Arc::clone(&self.sem),
7367 review_retries,
7368 &self.roles.reviewer_roster,
7369 &recon_ctx,
7370 &mut recon_quota_losses,
7371 &mut self.state,
7372 &|_: &ReviewRevote| Ok(()),
7373 )
7374 .await;
7375 if self.state.resource_block.is_some() {
7376 return self.stop_for_resource();
7377 }
7378 self.state.quota.extend(recon_quota_losses);
7379
7380 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7381 let agent_id = seat.agent.clone();
7382 self.state.seats.insert(seat.key.clone(), seat);
7383 let mut rec = ReviewRevoteRecord {
7384 reviewer: r + 1,
7385 agent: agent_id,
7386 vote: None,
7387 reason: String::new(),
7388 failed: None,
7389 };
7390 match res {
7391 Ok((rv, _)) => {
7392 rec.vote = Some(rv.vote);
7393 rec.reason =
7394 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7395 self.state.event(
7396 "review",
7397 format!(
7398 "round {round}: reviewer {} revoted {}",
7399 r + 1,
7400 rv.vote.label()
7401 ),
7402 );
7403 }
7404 Err(e) => {
7405 rec.failed = Some(e.to_string());
7406 self.state.event(
7407 "review",
7408 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7409 );
7410 }
7411 }
7412 reconsideration.push(rec);
7413 }
7414 } else if initial_votes.len() > 1 {
7415 self.state.event(
7416 "review",
7417 format!(
7418 "round {round}: votes agreed ({}) — no reconsideration",
7419 initial_votes[0].label()
7420 ),
7421 );
7422 }
7423
7424 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7425 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7426 let defer_e2e =
7437 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7438 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7439 let reason =
7440 format!("{blocking} blocking finding(s) already required a fix this round");
7441 self.state.event(
7442 "verify",
7443 format!(
7444 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7445 {}); it will run once a round has none left",
7446 short(&head)
7447 ),
7448 );
7449 (Vec::new(), false, true, Some(reason))
7450 } else {
7451 let e2e_commands = self.state.config.verify.e2e.clone();
7452 let cache_dir = self.state.config.cache_dir();
7453 let context = format!("round {round}");
7454 let wait = self.repository_wait();
7455 let (e2e, verify_retried) = with_cache_lease(
7456 &mut self.state,
7457 cache_dir.as_deref(),
7458 "e2e",
7459 "e2e",
7460 &winner.worktree,
7461 &head,
7462 verify_timeout,
7463 &context,
7464 &wait,
7465 |state, budget| {
7466 let shell = shell.clone();
7467 let e2e_commands = e2e_commands.clone();
7468 let worktree = winner.worktree.clone();
7469 let context = context.clone();
7470 async move {
7471 run_e2e_with_retry(
7472 state,
7473 &shell,
7474 &e2e_commands,
7475 &worktree,
7476 budget,
7477 &context,
7478 )
7479 .await
7480 }
7481 },
7482 )
7483 .await?;
7484 (e2e, verify_retried, false, None)
7485 };
7486
7487 let expected = records.len();
7488 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7489 let incomplete = answered < expected;
7490 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7491 let policy = self.state.config.graph.incomplete_review;
7492 let clean = round_is_clean(
7493 blocking,
7494 e2e_ok,
7495 answered,
7496 expected,
7497 round_quota_missing,
7498 policy,
7499 );
7500
7501 let mut round_record = ReviewRound {
7502 round,
7503 head: head.clone(),
7504 verified_head: None,
7505 verified_at: None,
7506 reviews: records,
7507 e2e,
7508 verify_retried,
7509 e2e_deferred,
7510 e2e_defer_reason,
7511 fix: None,
7512 blocking,
7513 answered,
7514 expected,
7515 clean,
7516 progressed: false,
7517 vote_split,
7518 reconsideration,
7519 verdict: None,
7520 };
7521 round_record.verdict = ReviewVote::worst(
7525 round_record
7526 .final_votes()
7527 .into_iter()
7528 .map(|(_, _, vote)| vote),
7529 );
7530 if !matches!(
7541 round_record.e2e_status(),
7542 E2eStatus::Deferred | E2eStatus::NotConfigured
7543 ) {
7544 round_record.verified_head = Some(head.clone());
7545 round_record.verified_at = Some(Timestamp::now());
7546 }
7547
7548 if incomplete {
7549 let missing: Vec<String> = round_record
7550 .reviews
7551 .iter()
7552 .filter(|r| r.failed.is_some())
7553 .map(|r| format!("review-{}", r.reviewer))
7554 .collect();
7555 self.state.event(
7556 "review",
7557 format!(
7558 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7559 missing.join(", ")
7560 ),
7561 );
7562 }
7563
7564 if clean {
7565 self.state.event(
7566 "review",
7567 if incomplete && policy == IncompleteReviewPolicy::Warn {
7568 format!(
7569 "round {round}: clean (warn policy, incomplete panel) — no \
7570 blocking findings from the seats that answered, verification green"
7571 )
7572 } else if incomplete {
7573 format!(
7574 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7575 quorum) — no blocking findings from the seats that answered, \
7576 verification green",
7577 expected - answered
7578 )
7579 } else {
7580 format!("round {round}: clean — no blocking findings, verification green")
7581 },
7582 );
7583 self.state.reviews.push(round_record);
7584 self.state.status = RunStatus::Gating;
7585 self.state.save()?;
7586 return Ok(());
7587 }
7588
7589 if incomplete && blocking == 0 && e2e_ok {
7597 self.state.reviews.push(round_record);
7598 self.state.save()?;
7599 if round == max_rounds {
7600 self.state.status = RunStatus::Blocked;
7601 self.state.event(
7602 "review",
7603 format!(
7604 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7605 refusing to call it clean",
7606 expected - answered
7607 ),
7608 );
7609 return Ok(());
7610 }
7611 continue;
7612 }
7613
7614 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7626 self.state.reviews.push(round_record);
7627 return self
7628 .stop_unresolved_review(
7629 "the round's own verification could not run",
7630 &shell,
7631 &winner.worktree,
7632 )
7633 .await;
7634 }
7635
7636 if round == max_rounds {
7637 self.state.reviews.push(round_record);
7638 return self
7639 .stop_reviewing(
7640 &format!(
7641 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7642 ),
7643 &shell,
7644 &winner.worktree,
7645 )
7646 .await;
7647 }
7648
7649 if !self.fix_review_round(&winner, round_record).await? {
7650 return Ok(());
7651 }
7652 }
7653 Ok(())
7654 }
7655
7656 async fn fix_review_round(
7659 &mut self,
7660 winner: &Candidate,
7661 mut round_record: ReviewRound,
7662 ) -> Result<bool> {
7663 let round = round_record.round;
7664 let shell = self.state.config.shell();
7665 let step = self.run_fix_round(winner, &mut round_record).await?;
7666 if matches!(step, FixStep::Resource) {
7667 self.stop_for_resource()?;
7668 return Ok(false);
7669 }
7670 if self.state.reviews.len() < round {
7671 self.state.reviews.push(round_record);
7672 } else {
7673 self.state.reviews[round - 1] = round_record;
7674 }
7675 self.state.save()?;
7676
7677 if matches!(step, FixStep::Recorded { report_lost: true }) {
7690 self.stop_reviewing(
7691 "the fixer's adoption report never came back, even after resuming its \
7692 own seat; refusing to start another round against the same worktree \
7693 while that is unresolved",
7694 &shell,
7695 &winner.worktree,
7696 )
7697 .await?;
7698 return Ok(false);
7699 }
7700
7701 let streak = self
7702 .state
7703 .reviews
7704 .iter()
7705 .rev()
7706 .take_while(|r| !r.progressed)
7707 .count();
7708 if streak >= STAGNANT_LIMIT {
7709 self.stop_reviewing(
7710 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7711 &shell,
7712 &winner.worktree,
7713 )
7714 .await?;
7715 return Ok(false);
7716 }
7717 Ok(true)
7718 }
7719
7720 async fn run_fix_round(
7722 &mut self,
7723 winner: &Candidate,
7724 round_record: &mut ReviewRound,
7725 ) -> Result<FixStep> {
7726 let round = round_record.round;
7727 let max_rounds = self.state.config.graph.review_rounds;
7728 let run_id = self.state.id.clone();
7729 let prompts = self.state.config.prompts.clone();
7730 let language = self.state.config.graph.language.clone();
7731 let sessions = self.state.config.graph.sessions;
7732 let artifacts = self.merge_repair_artifacts();
7733 let base = self.landing_base();
7734 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7735 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7736 let this_round_verification = round_record.verification_summary(&head);
7737 let all_findings: Vec<_> = round_record
7738 .reviews
7739 .iter()
7740 .flat_map(|r| r.findings.iter().cloned())
7741 .collect();
7742 let blocking_findings: Vec<_> = all_findings
7745 .iter()
7746 .filter(|f| f.severity.blocks())
7747 .cloned()
7748 .collect();
7749 let fix_prompt = prompt::fix_in(
7750 &self.state.instruction,
7751 &blocking_findings,
7752 this_round_verification.as_ref(),
7753 round,
7754 max_rounds,
7755 &language,
7756 &self.state.config.graph.github_language,
7757 );
7758 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7759 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7760 let (job, seat, out) = self
7761 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7762 prompt: fix_prompt.clone(),
7763 spec,
7764 seat,
7765 cwd: winner.worktree.clone(),
7766 timeout,
7767 allow_write: true,
7768 sessions,
7769 artifacts: artifacts.clone(),
7770 stem: format!("fix-{round}"),
7771 handover: None,
7772 })
7773 .await;
7774 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7777 return Ok(FixStep::Resource);
7778 }
7779 let agent_id = seat.agent.clone();
7780
7781 let mut fix = FixRecord {
7782 agent: agent_id,
7783 addressed: Vec::new(),
7784 rejected: Vec::new(),
7785 notes: String::new(),
7786 committed: false,
7787 failed: None,
7788 duration_ms: 0,
7789 continuation: None,
7790 };
7791 let mut continuation = ContinuationRecord::not_needed();
7792 let mut final_seat = seat.clone();
7793 match out {
7794 AgentOutcome::Ok(o) => {
7795 fix.duration_ms = o.duration_ms;
7796 let parsed = verdict::extract_json::<FixReport>(&o.text);
7797 let incomplete_reason = match &parsed {
7804 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7805 "the reply parsed, but it reported a command whose own CLI never \
7806 confirmed an exit status"
7807 .to_owned(),
7808 ),
7809 Ok(_) => None,
7810 Err(e) => Some(e.to_string()),
7811 };
7812 match incomplete_reason {
7813 None => {
7814 let report = parsed.expect("checked Ok above");
7815 fix.addressed = report.addressed;
7816 fix.rejected = report.rejected;
7817 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7818 }
7819 Some(reason) => {
7820 let (resumed_seat, resolved, failure, cont) = self
7821 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7822 .await;
7823 fix.duration_ms += cont.cumulative_wait_ms;
7824 continuation = cont;
7825 final_seat = resumed_seat;
7826 match resolved {
7827 Some(report) => {
7828 fix.addressed = report.addressed;
7829 fix.rejected = report.rejected;
7830 fix.notes =
7831 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7832 }
7833 None => fix.failed = failure,
7834 }
7835 }
7836 }
7837 }
7838 AgentOutcome::Dropped(o) => {
7840 fix.duration_ms = o.duration_ms;
7841 let why = o
7842 .dropped
7843 .as_ref()
7844 .map(|d| d.why.as_str())
7845 .unwrap_or("the CLI ended the stream without delivering its answer");
7846 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7847 }
7848 AgentOutcome::Quota(o) => {
7849 self.state.quota.push(QuotaLoss::from_report(
7850 final_seat.key.clone(),
7851 "fix",
7852 o.quota.as_ref(),
7853 ));
7854 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7855 }
7856 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7857 }
7858 if self.state.resource_block.is_some() {
7859 self.state.seats.insert(final_seat.key.clone(), final_seat);
7860 return Ok(FixStep::Resource);
7861 }
7862 fix.continuation = Some(continuation);
7863 self.state.seats.insert(final_seat.key.clone(), final_seat);
7864 let coordination = self.coordinate(&winner.worktree).await?;
7865 if let Ok(r) = git::rescue_commit(
7866 &winner.worktree,
7867 &format!("magi: review round {round} fixes (uncommitted work)"),
7868 )
7869 .await
7870 {
7871 self.state.note_withheld("fix", &r.withheld);
7872 }
7873 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7874 fix.committed = after != before;
7875 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7883 drop(coordination);
7884 let progressed = diff_after != patch;
7885 let commit_note = if fix.committed {
7886 "committed"
7887 } else {
7888 "NO new commit"
7889 };
7890 let tree_note = if progressed {
7891 "changed vs base"
7892 } else {
7893 "unchanged vs base"
7894 };
7895 self.state.event(
7896 "fix",
7897 match &fix.failed {
7898 Some(reason) => {
7904 format!(
7905 "round {round}: fixer's adoption report was lost ({reason}); \
7906 {commit_note}, tree {tree_note}"
7907 )
7908 }
7909 None => format!(
7910 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7911 {tree_note}{}",
7912 fix.addressed.len(),
7913 fix.rejected.len(),
7914 if continuation.outcome == ContinuationOutcome::Resumed {
7915 format!(
7916 " (adoption report recovered after {} continuation(s))",
7917 continuation.attempts
7918 )
7919 } else {
7920 String::new()
7921 },
7922 ),
7923 },
7924 );
7925 round_record.fix = Some(fix);
7926 round_record.progressed = progressed;
7927 Ok(FixStep::Recorded {
7928 report_lost: matches!(
7929 continuation.outcome,
7930 ContinuationOutcome::Exhausted
7931 | ContinuationOutcome::QuotaLost
7932 | ContinuationOutcome::NoSession
7933 ),
7934 })
7935 }
7936
7937 fn record_contested_handoff(&mut self) {
7971 if self.state.contested_handoff.is_some() {
7972 return;
7973 }
7974 let Some(contested) = self
7975 .state
7976 .reviews
7977 .last()
7978 .and_then(ReviewRound::contested_handoff)
7979 else {
7980 return;
7981 };
7982 self.state.event(
7983 "review",
7984 format!(
7985 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7986 wait for the owner's approval",
7987 contested.findings.len(),
7988 contested.rejecters.len()
7989 ),
7990 );
7991 self.state.contested_handoff = Some(contested);
7992 }
7993
7994 async fn stop_unresolved_review(
7995 &mut self,
7996 why: &str,
7997 shell: &[String],
7998 worktree: &Path,
7999 ) -> Result<()> {
8000 self.stop_reviewing_inner(why, shell, worktree, true).await
8001 }
8002
8003 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8004 self.stop_reviewing_inner(why, shell, worktree, false).await
8005 }
8006
8007 async fn stop_reviewing_inner(
8008 &mut self,
8009 why: &str,
8010 shell: &[String],
8011 worktree: &Path,
8012 repairable: bool,
8013 ) -> Result<()> {
8014 let round_idx = self.state.reviews.len() - 1;
8015 let needs_catchup_run = matches!(
8023 self.state.reviews[round_idx].e2e_status(),
8024 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8025 );
8026 if needs_catchup_run {
8027 let round = self.state.reviews[round_idx].round;
8028 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8029 let commands = self.state.config.verify.e2e.clone();
8030 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8031 let cache_dir = self.state.config.cache_dir();
8032 let context = format!(
8033 "round {round}: verification unresolved, catching up before the final decision"
8034 );
8035 let wait = self.repository_wait();
8036 let (outcomes, verify_retried) = with_cache_lease(
8037 &mut self.state,
8038 cache_dir.as_deref(),
8039 "e2e",
8040 "e2e",
8041 worktree,
8042 &attempted_head,
8043 timeout,
8044 &context,
8045 &wait,
8046 |state, budget| {
8047 let shell = shell.to_vec();
8048 let commands = commands.clone();
8049 let context = context.clone();
8050 async move {
8051 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8052 .await
8053 }
8054 },
8055 )
8056 .await?;
8057 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8058 self.state.record_base_progress(&attempted_head);
8059 }
8060 let last = &mut self.state.reviews[round_idx];
8061 last.e2e = outcomes;
8062 last.verify_retried = verify_retried;
8063 last.verified_head = Some(attempted_head);
8070 last.verified_at = Some(Timestamp::now());
8071 if verify_inconclusive(&last.e2e) {
8072 self.state.save()?;
8079 return Ok(());
8080 }
8081 last.e2e_deferred = false;
8082 if last.answered == last.expected {
8087 last.clean = round_is_clean(
8088 last.blocking,
8089 last.e2e.iter().all(CommandOutcome::ok),
8090 last.answered,
8091 last.expected,
8092 0,
8093 self.state.config.graph.incomplete_review,
8094 );
8095 }
8096 }
8097 let last = &self.state.reviews[round_idx];
8098 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8099
8100 match last.e2e_status() {
8101 E2eStatus::Failed => {
8102 let red: Vec<String> = last
8103 .e2e
8104 .iter()
8105 .filter(|o| !o.ok())
8106 .map(|o| {
8107 format!(
8108 "`{}` -> {:?}\n{}",
8109 o.command,
8110 o.code,
8111 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8112 )
8113 })
8114 .collect();
8115 let can_repair = last.blocking == 0 && last.fix.is_none();
8116 self.state.reviews[round_idx].clean = false;
8117 self.state
8118 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8119 self.state.status = RunStatus::Blocked;
8120 if repairable
8121 && can_repair
8122 && self.state.reviews.len() < self.state.config.graph.review_rounds
8123 {
8124 self.state.status = RunStatus::Reviewing;
8125 }
8126 }
8127 E2eStatus::ResourceBlocked => {
8132 self.state.event(
8133 "review",
8134 format!(
8135 "{why}; e2e could not run (resource unavailable); not \
8136 deciding yet"
8137 ),
8138 );
8139 }
8140 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8141 self.state.event(
8142 "review",
8143 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8144 );
8145 self.record_contested_handoff();
8146 self.state.status = RunStatus::Gating;
8147 }
8148 }
8149 self.state.save()?;
8150 Ok(())
8151 }
8152
8153 async fn gate(&mut self) -> Result<()> {
8156 if self.hold_base_sync().await? {
8157 return Ok(());
8158 }
8159 if self.state.status == RunStatus::Failed
8171 || self
8172 .state
8173 .base_sync
8174 .as_ref()
8175 .is_some_and(|s| s.conflict.is_some())
8176 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8177 != Some(RunStatus::Gating)
8178 {
8179 return Ok(());
8180 }
8181 let Some(winner) = self.state.winner().cloned() else {
8182 return Ok(());
8183 };
8184 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8185 self.state.reconcile_verification_head(&head);
8186 self.state.save()?;
8187 let wait = self.repository_wait();
8188 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8189 return Ok(());
8190 }
8191 if self.state.gate_ran {
8192 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8203 self.state.status = RunStatus::Blocked;
8204 self.state.save()?;
8205 }
8206 return Ok(());
8207 }
8208 self.state.status = RunStatus::Gating;
8209 let mut outcomes = self.run_gate(&winner).await?;
8210 let mut attempted_head = self.state.gate_head.clone();
8211 loop {
8212 if verify_inconclusive(&outcomes) {
8223 self.state.save()?;
8224 return Ok(());
8225 }
8226 if outcomes.iter().all(CommandOutcome::ok) {
8227 break;
8228 }
8229 let fixes_before = self.state.gate_fixes.len();
8230 match self.gate_fix_round(&winner, &outcomes).await? {
8231 GateFix::Retry => {
8232 outcomes = self.run_gate(&winner).await?;
8233 attempted_head = self.state.gate_head.clone();
8234 }
8235 GateFix::Stop => {
8236 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8237 if self.state.gate_fixes.len() > fixes_before
8238 && attempted_head.as_deref() != Some(current.as_str())
8239 && verify_current_head(&mut self.state, &winner, &wait).await?
8240 {
8241 outcomes = self.run_gate(&winner).await?;
8244 attempted_head = self.state.gate_head.clone();
8245 } else {
8246 break;
8247 }
8248 }
8249 GateFix::Defer => {
8250 self.state.save()?;
8251 return Ok(());
8252 }
8253 }
8254 }
8255 let passed = outcomes.iter().all(CommandOutcome::ok);
8256 self.state.gate = outcomes;
8257 self.state.gate_ran = true;
8258 self.state.gate_head = attempted_head;
8259 if !self.state.gate.is_empty()
8260 && !verify_inconclusive(&self.state.gate)
8261 && let Some(head) = self.state.gate_head.clone()
8262 {
8263 self.state.record_base_progress(&head);
8264 }
8265 self.state.save()?;
8266 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8267 return Ok(());
8268 }
8269 if !passed {
8270 self.state.status = RunStatus::Blocked;
8271 let spent = self.state.gate_fixes.len();
8272 self.state.event(
8273 "gate",
8274 if spent == 0 {
8275 "gate failed; not merging".to_owned()
8276 } else {
8277 format!("gate failed after {spent} gate-fix round(s); not merging")
8278 },
8279 );
8280 }
8281 self.state.save()?;
8282 Ok(())
8283 }
8284
8285 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8295 let commands = state.config.verify.pre_gate.clone();
8296 if commands.is_empty() {
8297 return;
8298 }
8299 let shell = state.config.shell();
8300 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8301 let (outcomes, _) = run_commands(
8302 state,
8303 "pre_gate",
8304 "pre_gate",
8305 0,
8306 &shell,
8307 &commands,
8308 &winner.worktree,
8309 timeout,
8310 )
8311 .await;
8312 for o in &outcomes {
8313 if !o.ok() {
8314 tracing::warn!(
8315 "pre_gate `{}` failed ({:?}); the gate decides",
8316 o.command,
8317 o.code
8318 );
8319 }
8320 state.event(
8321 "pre_gate",
8322 format!(
8323 "`{}` -> {}",
8324 o.command,
8325 if o.ok() {
8326 "pass".to_owned()
8327 } else {
8328 format!(
8329 "FAIL ({:?})\n{}",
8330 o.code,
8331 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8332 )
8333 }
8334 ),
8335 );
8336 }
8337 state.pre_gate = outcomes;
8338 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8339 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8340 Ok(head) => {
8341 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8342 state.pre_gate_commit = Some(head);
8343 }
8344 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8345 },
8346 Ok(false) => {}
8347 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8348 }
8349 if let Err(e) = state.save() {
8350 tracing::warn!("could not persist the pre_gate record: {e:#}");
8351 }
8352 }
8353
8354 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8357 let wait = self.repository_wait();
8358 run_landing_gate(&mut self.state, winner, &wait).await
8359 }
8360
8361 async fn gate_fix_round(
8374 &mut self,
8375 winner: &Candidate,
8376 outcomes: &[CommandOutcome],
8377 ) -> Result<GateFix> {
8378 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8379 g.phase == crate::repair_budget::Phase::Fixing
8380 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8381 }) {
8382 self.state.gate_fixes.len().saturating_add(1)
8383 } else {
8384 self.state.config.graph.gate_fix_rounds
8385 };
8386 let spent = self.state.gate_fixes.len();
8387 if spent >= cap {
8388 if cap > 0 {
8389 self.state.event(
8390 "gate",
8391 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8392 );
8393 }
8394 return Ok(GateFix::Stop);
8395 }
8396 if !gate_fixable(outcomes) {
8397 self.state.event(
8398 "gate",
8399 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8400 command or similar); not spending a fix round on it",
8401 );
8402 return Ok(GateFix::Stop);
8403 }
8404 let min_free = self.state.config.disk.min_free_bytes;
8405 if min_free > 0 {
8406 match crate::disk::free_bytes(&winner.worktree) {
8407 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8408 Ok(free) => {
8409 self.state.event(
8410 "gate",
8411 format!(
8412 "only {free} bytes free ({min_free} required by `[disk] \
8413 min_free_bytes`); not spending a fix round on a failure the disk \
8414 may explain"
8415 ),
8416 );
8417 return Ok(GateFix::Stop);
8418 }
8419 Err(e) => {
8420 self.state.event(
8421 "gate",
8422 format!("free disk space could not be measured ({e:#}); no fix round"),
8423 );
8424 return Ok(GateFix::Stop);
8425 }
8426 }
8427 }
8428
8429 let attempt = spent + 1;
8430 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8431 let base = self.landing_base();
8432 let fix_prompt = prompt::gate_fix_in(
8433 &self.state.instruction,
8434 &failed,
8435 attempt,
8436 cap,
8437 &self.state.config.graph.language,
8438 &self.state.config.graph.github_language,
8439 );
8440 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8441 let sessions = self.state.config.graph.sessions;
8442 let artifacts = self.merge_repair_artifacts();
8443 self.state.event(
8444 "gate",
8445 format!("gate failed; gate-fix round {attempt} of {cap}"),
8446 );
8447 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8448 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8449 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8452 head: before.clone(),
8453 passed: false,
8454 outcomes: Vec::new(),
8455 });
8456 self.state.save()?;
8457 let (_, seat, out) = self
8458 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8459 prompt: fix_prompt.clone(),
8460 spec,
8461 seat,
8462 cwd: winner.worktree.clone(),
8463 timeout,
8464 allow_write: true,
8465 sessions,
8466 artifacts: artifacts.clone(),
8467 stem: format!("gate-fix-{attempt}"),
8468 handover: None,
8469 })
8470 .await;
8471 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8474 return Ok(GateFix::Defer);
8475 }
8476 let mut record = GateFixRecord {
8477 agent: seat.agent.clone(),
8478 failed,
8479 notes: String::new(),
8480 committed: false,
8481 error: None,
8482 };
8483 match out {
8484 AgentOutcome::Ok(o) => {
8485 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8488 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8489 }
8490 }
8491 AgentOutcome::Dropped(_) => {
8492 record.error = Some("the CLI dropped the stream".to_owned());
8493 }
8494 AgentOutcome::Quota(o) => {
8495 self.state.quota.push(QuotaLoss::from_report(
8496 seat.key.clone(),
8497 "gate-fix",
8498 o.quota.as_ref(),
8499 ));
8500 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8501 }
8502 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8503 }
8504 self.state.seats.insert(seat.key.clone(), seat);
8505 let coordination = self.coordinate(&winner.worktree).await?;
8506 if let Ok(r) = git::rescue_commit(
8507 &winner.worktree,
8508 &format!("magi: gate fix {attempt} (uncommitted work)"),
8509 )
8510 .await
8511 {
8512 self.state.note_withheld("gate-fix", &r.withheld);
8513 }
8514 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8515 record.committed = after != before;
8516 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8517 let note = record.error.clone();
8518 self.state.gate_fixes.push(record);
8519 self.state.save()?;
8520 if !changed {
8521 self.state.event(
8522 "gate",
8523 match note {
8524 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8525 None => format!("gate-fix round {attempt}: the tree did not change"),
8526 },
8527 );
8528 return Ok(GateFix::Stop);
8529 }
8530 self.state.event(
8531 "gate",
8532 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8533 );
8534
8535 drop(coordination);
8536 let wait = self.repository_wait();
8537 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8538 }
8539
8540 async fn guarded_pr_message(
8547 &mut self,
8548 winner: &Candidate,
8549 facts: Option<&BranchFacts>,
8550 posting: bool,
8551 ) -> Result<Option<PrMessage>> {
8552 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8553 let head = match final_head.as_deref() {
8554 Some(h) => crate::pr_outcome::FinalHead::At(h),
8555 None => crate::pr_outcome::FinalHead::Unknown,
8556 };
8557 let summarized = if posting {
8559 Some(self.summarize_change(winner, final_head.as_deref()).await)
8560 } else {
8561 None
8562 };
8563 if self.state.resource_block.is_some() {
8568 self.state.event(
8569 "pr-summary",
8570 "a shared resource was busy; nothing was published, resume to summarize again",
8571 );
8572 return Ok(None);
8573 }
8574 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8575 anyhow::bail!(
8576 "the final commit moved during public description generation; retry before publishing"
8577 );
8578 }
8579 let note = match (&summarized, &self.state.change_summary) {
8580 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8581 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8582 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8583 (None, _) => ChangeNote::NotAsked,
8584 };
8585 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8586 if !posting {
8587 let identity = crate::scrub::Identity::current();
8588 return Ok(Some(PrMessage {
8589 title: crate::scrub::scrub(&pr.title, &identity),
8590 body: crate::scrub::scrub(&pr.body, &identity),
8591 }));
8592 }
8593 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8596 if let Some(known) = self
8597 .state
8598 .github_text
8599 .clone()
8600 .filter(|g| g.fingerprint == raw_fp)
8601 {
8602 return self.settle_withheld(winner, &pr, known).await;
8603 }
8604 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8605 let mut violations =
8606 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8607 let identity = crate::scrub::Identity::current();
8608 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8609 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8610 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8611 {
8612 violations.push(crate::github_text::Violation::SensitiveData);
8613 }
8614 let has_language = violations
8618 .iter()
8619 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8620 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8621 self.state.event(
8622 "github-text",
8623 if has_language {
8624 format!("description rejected: {violations:?}; requesting one rewrite")
8625 } else {
8626 format!("description rejected: {violations:?}; asking the owner")
8627 },
8628 );
8629 let mut rewritten = false;
8630 let mut rejected_rewrite = None;
8631 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8632 {
8633 let output = text;
8634 if let Some(title) = summary_title(&output) {
8635 let raw_body = summary_without_title(&output);
8638 let clean_title = crate::scrub::scrub(&title, &identity);
8639 let mut body = crate::scrub::scrub(&raw_body, &identity);
8640 let redacted = clean_title != title || body != raw_body;
8641 let title = clean_title;
8642 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8643 if !complete {
8644 self.state.event(
8645 "github-text",
8646 "description rewrite omitted required findings",
8647 );
8648 }
8649 if !body.trim().is_empty() {
8650 body.push_str("\n\n");
8651 body.push_str(&public_pr_evidence(&self.state, head));
8652 }
8653 let marker = format!("magi:run/{}", self.state.id);
8654 if !body.trim().is_empty() && !body.contains(&marker) {
8655 body.push_str(&format!("\n\n{marker}"));
8656 }
8657 let rewrite = PrMessage {
8658 title: title.clone(),
8659 body: body.clone(),
8660 };
8661 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8662 let mut failed = crate::github_text::check_for(
8663 &self.state.config.graph,
8664 &title,
8665 &body,
8666 redecision,
8667 );
8668 if (!crate::github_text::shareable(&title)
8669 || !crate::github_text::shareable(&body))
8670 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8671 {
8672 failed.push(crate::github_text::Violation::SensitiveData);
8673 }
8674 if complete
8675 && !title.trim().is_empty()
8676 && !body.trim().is_empty()
8677 && !failed.is_empty()
8678 {
8679 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8680 title: title.clone(),
8681 body: body.clone(),
8682 failed: failed
8683 .iter()
8684 .map(|v| crate::github_text::category(*v).to_owned())
8685 .collect(),
8686 });
8687 }
8688 if complete
8689 && !title.trim().is_empty()
8690 && !raw_body.trim().is_empty()
8691 && failed.is_empty()
8692 {
8693 decision = redecision;
8694 pr = PrMessage { title, body };
8695 rewritten = true;
8696 self.state
8697 .event("github-text", "description rewrite passed");
8698 if redacted {
8699 self.state.event(
8700 "github-text",
8701 "sensitive data removed from description rewrite",
8702 );
8703 }
8704 }
8705 }
8706 }
8707 if !rewritten {
8708 self.state.event(
8709 "github-text",
8710 "description rewrite unavailable or rejected; asking the owner",
8711 );
8712 if let Some(w) =
8713 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8714 {
8715 self.retire_github_text_question("the withheld text changed");
8718 let record = crate::run::GithubTextAsk {
8719 rewrite: rejected_rewrite,
8720 fingerprint: raw_fp,
8721 title: w.title,
8722 body: w.body,
8723 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8724 question: None,
8725 asks: 0,
8726 resolved: false,
8727 chosen_title: None,
8728 choice: None,
8729 };
8730 return self.settle_withheld(winner, &pr, record).await;
8731 }
8732 }
8733 }
8734 let (title, body) =
8735 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8736 Ok(Some(PrMessage { title, body }))
8737 }
8738
8739 async fn summarize_change(
8750 &mut self,
8751 winner: &Candidate,
8752 head: Option<&str>,
8753 ) -> std::result::Result<(), String> {
8754 let Some(head) = head else {
8755 return Err("the final commit could not be read".to_owned());
8756 };
8757 let remote = self.state.config.merge.remote.clone();
8758 let tracking = format!("{remote}/{}", self.state.base_branch);
8759 if !git::rev_exists(&winner.worktree, &tracking).await {
8760 return Err(format!("the remote base `{tracking}` could not be read"));
8761 }
8762 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8763 Ok(b) if !b.is_empty() => b,
8764 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8765 };
8766 let evidence = crate::pr_outcome::summary_evidence(
8767 &self.state,
8768 crate::pr_outcome::FinalHead::At(head),
8769 );
8770 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8771 if self
8772 .state
8773 .change_summary
8774 .as_ref()
8775 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8776 {
8777 return Ok(());
8778 }
8779 if self.state.change_summary.take().is_some() {
8780 self.state.event(
8781 "pr-summary",
8782 "the final diff or evidence changed; the earlier change summary was discarded",
8783 );
8784 }
8785 if let Err(e) = write_artifact(
8787 &self.state,
8788 &format!("public-description-{fingerprint}.md"),
8789 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8790 ) {
8791 tracing::warn!("could not save public description evidence: {e:#}");
8792 }
8793 let chain = match agent::pick_chain(
8794 &self.state.config.agents,
8795 self.state.config.roles.synthesizer.as_ref(),
8796 &agent::installed,
8797 "synthesizer",
8798 ) {
8799 Ok(chain) => chain,
8800 Err(e) => {
8801 tracing::warn!("no agent for the change summary: {e:#}");
8802 self.state.event("pr-summary", "no summary agent available");
8803 return Err("no agent was available to summarize it".to_owned());
8804 }
8805 };
8806 let subjects = git::subjects(&winner.worktree, &base, head)
8807 .await
8808 .unwrap_or_default();
8809 let stat = git::diff_stat(&winner.worktree, &base, head)
8810 .await
8811 .unwrap_or_default();
8812 let stat: String = stat.chars().take(6_000).collect();
8813 let mut prompt = crate::prompt::change_summary_in(
8814 &base,
8815 head,
8816 &subjects,
8817 stat.trim(),
8818 &self.state.config.graph.github_language,
8819 );
8820 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8821 let sessions = self.state.config.graph.sessions;
8822 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8823 let prompts = self.state.config.prompts.clone();
8824 let cache = self.state.config.cache_dir();
8825 let run = self.state.id.clone();
8826 for (i, spec) in chain.into_iter().enumerate() {
8827 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8828 let github_spec = match agent::for_github_text(
8829 &spec,
8830 &self.state.config.graph.github_language,
8831 &winner.worktree,
8832 ) {
8833 Ok(spec) => spec,
8834 Err(e) => {
8835 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8836 continue;
8837 }
8838 };
8839 let job = SeatJob {
8840 spec: github_spec,
8841 seat,
8842 cwd: winner.worktree.clone(),
8843 prompt: prompt.clone(),
8844 timeout,
8845 allow_write: false,
8846 sessions,
8847 artifacts: agent::artifacts_dir(&self.state.dir()),
8848 stem: if i == 0 {
8849 "change-summary".to_owned()
8850 } else {
8851 format!("change-summary-{}", spec.id)
8852 },
8853 handover: None,
8854 };
8855 let ctx = WaveCtx {
8856 run: &run,
8857 node: "pr-summary",
8858 prompts: &prompts,
8859 cache: cache.as_deref(),
8860 round: None,
8861 carry_seats: false,
8862 pauses: Some((&self.pause, &self.interrupt)),
8863 };
8864 let (seat, outcome) =
8865 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8866 self.state.seats.insert(seat.key.clone(), seat);
8867 let text = match outcome {
8868 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8869 AgentOutcome::ResourceBlocked(why) => {
8870 if self.state.resource_block.is_none() {
8873 self.state.resource_block = Some(crate::run::ResourceBlock {
8874 cause: crate::run::ResourceBlockCause::Other,
8875 node: "pr-summary".to_owned(),
8876 kind: "shared resource".to_owned(),
8877 detail: why,
8878 at: Timestamp::now(),
8879 });
8880 }
8881 return Err("a shared resource was busy".to_owned());
8882 }
8883 _ => continue,
8884 };
8885 let blind = &self.state.config.blind;
8886 let Some(title) = summary_title(&text) else {
8887 continue;
8888 };
8889 let body: String = summary_without_title(&text)
8890 .lines()
8891 .filter(|l| !l.contains("magi:run/"))
8892 .collect::<Vec<_>>()
8893 .join("\n");
8894 let body = blind::sanitize_prose(body.trim(), blind);
8895 let title = blind::sanitize_prose(&title, blind);
8896 if body.trim().is_empty() || title.trim().is_empty() {
8897 continue;
8898 }
8899 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8900 continue;
8901 }
8902 if git::rev_parse(&winner.worktree, "HEAD")
8904 .await
8905 .ok()
8906 .as_deref()
8907 != Some(head)
8908 {
8909 return Err("the branch moved while the summary was written".to_owned());
8910 }
8911 if git::merge_base(&winner.worktree, &tracking, head)
8912 .await
8913 .ok()
8914 .as_deref()
8915 != Some(base.as_str())
8916 || crate::pr_outcome::summary_evidence(
8917 &self.state,
8918 crate::pr_outcome::FinalHead::At(head),
8919 ) != evidence
8920 {
8921 return Err("the summary evidence changed while it was written".to_owned());
8922 }
8923 self.state.change_summary = Some(crate::run::ChangeSummary {
8924 head: head.to_owned(),
8925 base,
8926 evidence: fingerprint,
8927 title,
8928 body,
8929 });
8930 self.state.event(
8931 "pr-summary",
8932 format!("change summary written by {} for the final diff", spec.id),
8933 );
8934 if let Err(e) = self.state.save() {
8935 tracing::warn!("could not save the change summary: {e:#}");
8936 }
8937 return Ok(());
8938 }
8939 self.state
8940 .event("pr-summary", "no agent produced a change summary");
8941 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8942 }
8943
8944 async fn rewrite_pr_text(
8954 &mut self,
8955 winner: &Candidate,
8956 pr: &PrMessage,
8957 violations: &[crate::github_text::Violation],
8958 ) -> Option<String> {
8959 let prompt = format!(
8960 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8961 pr.title, pr.body
8962 );
8963 let sessions = self.state.config.graph.sessions;
8964 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8965 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8968 let key = format!("impl-{}", winner.label);
8969 if let Some(spec) = self
8970 .state
8971 .config
8972 .agents
8973 .iter()
8974 .find(|a| a.id == winner.agent)
8975 .cloned()
8976 && let Some(existing) = self.state.seats.get(&key)
8977 && existing.agent == spec.id
8978 && agent::has_session(spec.kind, existing, sessions)
8979 {
8980 attempts.push((spec, Some(existing.clone())));
8981 }
8982 match agent::pick_chain(
8983 &self.state.config.agents,
8984 self.state.config.roles.language_judge.as_ref(),
8985 &agent::installed,
8986 "language_judge",
8987 ) {
8988 Ok(chain) => {
8989 for spec in chain {
8990 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8991 continue;
8992 }
8993 attempts.push((spec, None));
8994 }
8995 }
8996 Err(e) if attempts.is_empty() => {
8997 tracing::warn!("no agent for the description rewrite: {e:#}");
8998 self.state
8999 .event("github-text", "no rewrite agent available");
9000 return None;
9001 }
9002 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9003 }
9004 let prompts = self.state.config.prompts.clone();
9005 let cache = self.state.config.cache_dir();
9006 let run = self.state.id.clone();
9007 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9008 let resumed = resumed_seat.is_some();
9009 let seat = resumed_seat.unwrap_or_else(|| {
9010 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9011 });
9012 let github_spec = match agent::for_github_text(
9013 &spec,
9014 &self.state.config.graph.github_language,
9015 &winner.worktree,
9016 ) {
9017 Ok(spec) => spec,
9018 Err(e) => {
9019 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9020 continue;
9021 }
9022 };
9023 let job = SeatJob {
9024 spec: github_spec,
9025 seat,
9026 cwd: winner.worktree.clone(),
9027 prompt: prompt.clone(),
9028 timeout: retry_budget(timeout_full, resumed),
9029 allow_write: false,
9030 sessions,
9031 artifacts: agent::artifacts_dir(&self.state.dir()),
9032 stem: if i == 0 {
9033 "github-text-rewrite".to_owned()
9034 } else {
9035 format!("github-text-rewrite-{}", spec.id)
9036 },
9037 handover: None,
9038 };
9039 let ctx = WaveCtx {
9040 run: &run,
9041 node: "github-text",
9042 prompts: &prompts,
9043 cache: cache.as_deref(),
9044 round: None,
9045 carry_seats: false,
9046 pauses: Some((&self.pause, &self.interrupt)),
9047 };
9048 let (seat, outcome) =
9049 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9050 self.state.seats.insert(seat.key.clone(), seat);
9051 match outcome {
9052 AgentOutcome::Ok(o)
9053 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9054 {
9055 return Some(o.text);
9056 }
9057 AgentOutcome::ResourceBlocked(_) => return None,
9058 _ => {}
9059 }
9060 }
9061 None
9062 }
9063
9064 fn retire_github_text_question(&mut self, why: &str) {
9066 let Some(id) = self
9067 .state
9068 .github_text
9069 .as_ref()
9070 .and_then(|g| g.question.clone())
9071 else {
9072 return;
9073 };
9074 let _ = ask::Questions::open().update(&id, |q| {
9075 q.abandon(why);
9076 Ok(())
9077 });
9078 }
9079
9080 async fn settle_withheld(
9084 &mut self,
9085 winner: &Candidate,
9086 pr: &PrMessage,
9087 mut g: crate::run::GithubTextAsk,
9088 ) -> Result<Option<PrMessage>> {
9089 use crate::github_text as gt;
9090 let store = ask::Questions::open();
9091 let timeout = self.state.config.graph.answer_timeout;
9092 if !g.resolved {
9093 let mut standing = g
9094 .question
9095 .as_ref()
9096 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9097 if let Some(q) = standing.as_ref()
9098 && q.status.open()
9099 && gt::expired(q, timeout)
9100 {
9101 let why = format!("no answer within {}s of asking", timeout.max(1));
9102 standing = store
9103 .update(&q.id, |q| {
9104 q.abandon(&why);
9105 Ok(())
9106 })
9107 .ok()
9108 .map(|(q, ())| q);
9109 }
9110 let mut ask_again: Option<Option<Vec<String>>> = None;
9111 match standing {
9112 None if g.asks <= 1 => {
9115 g.asks = 0;
9116 ask_again = Some(None);
9117 }
9118 None => {
9119 g.resolved = true;
9120 g.chosen_title = None;
9121 g.choice = Some(GithubTextChoice::Fallback);
9122 }
9123 Some(q) if q.status.open() => {
9124 self.park_on_github_text(&g)?;
9125 return Ok(None);
9126 }
9127 Some(q) => match gt::read_reply(&q) {
9128 gt::Reply::Fallback => {
9129 g.resolved = true;
9130 g.chosen_title = None;
9131 g.choice = Some(GithubTextChoice::Fallback);
9132 }
9133 gt::Reply::Original => {
9134 g.resolved = true;
9135 g.chosen_title = None;
9136 g.choice = Some(GithubTextChoice::Original);
9137 }
9138 gt::Reply::Rewrite => {
9139 g.resolved = true;
9140 g.chosen_title = None;
9141 g.choice = Some(if g.rewrite.is_some() {
9142 GithubTextChoice::Rewrite
9143 } else {
9144 GithubTextChoice::Fallback
9145 });
9146 }
9147 gt::Reply::Title(text) => {
9148 let first = text
9149 .lines()
9150 .map(str::trim)
9151 .find(|l| !l.is_empty())
9152 .unwrap_or("");
9153 let decision = if first.is_empty() {
9154 None
9155 } else {
9156 let candidate = PrMessage {
9157 title: first.to_owned(),
9158 body: String::new(),
9159 };
9160 self.judge_pr_language(&winner.worktree, &candidate).await
9161 };
9162 match gt::vet_title(
9163 &text,
9164 decision,
9165 self.state.config.graph.github_english(),
9166 ) {
9167 Ok(title) => {
9168 g.resolved = true;
9169 g.chosen_title = Some(title);
9170 g.choice = Some(GithubTextChoice::MyTitle);
9171 }
9172 Err(cats) if g.asks < 2 => {
9173 ask_again = Some(Some(
9174 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9175 ))
9176 }
9177 Err(_) => {
9178 g.resolved = true;
9179 g.chosen_title = None;
9180 g.choice = Some(GithubTextChoice::Fallback);
9181 }
9182 }
9183 }
9184 },
9185 }
9186 if let Some(rejected) = ask_again {
9187 let w = gt::Withheld {
9190 title: g.title,
9191 body: g.body,
9192 categories: g.categories.clone(),
9193 };
9194 let artifact = match crate::run::write_artifact(
9197 &self.state,
9198 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9199 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9200 ) {
9201 Ok(p) => Some(p),
9202 Err(e) => {
9203 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9204 None
9205 }
9206 };
9207 let (detail, comparison) = gt::question_content(
9208 &w,
9209 &pr.title,
9210 &pr.body,
9211 &self.state.id,
9212 rejected.as_deref(),
9213 artifact.as_deref(),
9214 g.rewrite.as_ref(),
9215 );
9216 let mut q = ask::Question::new(
9217 self.state.id.clone(),
9218 gt::ASK_NODE.to_owned(),
9219 gt::ASK_SEAT.to_owned(),
9220 gt::question_summary(&self.state.config.graph.language, &w),
9221 detail,
9222 gt::question_choices(&w, g.rewrite.as_ref()),
9223 );
9224 q.posting_comparison = Some(comparison);
9225 q.answer_timeout = timeout;
9226 g.question = Some(q.id.clone());
9229 g.asks += 1;
9230 self.state.github_text = Some(g.clone());
9231 self.state.event(
9232 "github-text",
9233 format!("asking the owner about the withheld text ({})", q.short()),
9234 );
9235 self.state.save()?;
9236 store
9237 .put_executable(&mut q)
9238 .context("file the posting-gate question")?;
9239 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9240 tracing::warn!(
9241 "could not notify about posting-gate question {}: {e:#}",
9242 q.short()
9243 );
9244 }
9245 self.park_on_github_text(&g)?;
9246 return Ok(None);
9247 }
9248 }
9249 let resolved = g.chosen_title.clone();
9250 let original = g.choice == Some(GithubTextChoice::Original);
9251 self.state.event(
9253 "github-text",
9254 if original {
9255 format!(
9256 "posting the original text for: {}",
9257 match (g.title, g.body) {
9258 (true, true) => "title, description",
9259 (true, false) => "title",
9260 _ => "description",
9261 }
9262 )
9263 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9264 "posting the saved rewrite title and description".to_owned()
9265 } else if resolved.is_some() {
9266 "posting the owner's replacement title".to_owned()
9267 } else {
9268 "posting the neutral text for the withheld fields".to_owned()
9269 },
9270 );
9271 self.state.github_text = Some(g.clone());
9272 self.state.save()?;
9273 if g.choice == Some(GithubTextChoice::Rewrite)
9274 && let Some(r) = g.rewrite.as_ref()
9275 {
9276 return Ok(Some(PrMessage {
9277 title: r.title.clone(),
9278 body: r.body.clone(),
9279 }));
9280 }
9281 let w = gt::Withheld {
9282 title: g.title,
9283 body: g.body,
9284 categories: g.categories.clone(),
9285 };
9286 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9287 if let Some(t) = resolved {
9288 title = t;
9289 }
9290 let english = gt::LanguageDecision {
9292 title_english: true,
9293 body_english: true,
9294 body_complete: true,
9295 };
9296 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9297 if original {
9298 if g.title {
9301 title = pr.title.clone();
9302 }
9303 if g.body {
9304 body = pr.body.clone();
9305 }
9306 }
9307 Ok(Some(PrMessage { title, body }))
9308 }
9309
9310 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9311 self.state.github_text = Some(g.clone());
9312 self.state.end_pass_parked(ParkCause::Question);
9313 self.state.parked = true;
9314 self.state.event(
9315 "github-text",
9316 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9317 );
9318 self.state.save()
9319 }
9320
9321 async fn judge_pr_language(
9324 &mut self,
9325 cwd: &Path,
9326 pr: &PrMessage,
9327 ) -> Option<crate::github_text::LanguageDecision> {
9328 self.state.config.roles.language_judge.as_ref()?;
9329 if !self.state.config.graph.language_gate() {
9330 return None;
9331 }
9332 let decision =
9333 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9334 self.state.event(
9335 "github-text",
9336 if decision.is_some() {
9337 "language decided by the language judge"
9338 } else {
9339 "language judge unavailable; using built-in heuristics"
9340 },
9341 );
9342 decision
9343 }
9344
9345 async fn merge(&mut self) -> Result<()> {
9346 if self.resource_halt()? || self.hold_base_sync().await? {
9347 return Ok(());
9348 }
9349 if self
9364 .state
9365 .base_sync
9366 .as_ref()
9367 .is_some_and(|s| s.conflict.is_some())
9368 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9369 != Some(RunStatus::Gating)
9370 || !self.state.gate_allows_progress()
9378 {
9379 return Ok(());
9380 }
9381 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9390 return Ok(());
9391 }
9392 let Some(winner) = self.state.winner().cloned() else {
9393 return Ok(());
9394 };
9395 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9396 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9397 self.state.status = RunStatus::Blocked;
9398 self.state.event(
9399 "gate",
9400 "winner changed after gate; fresh verification required",
9401 );
9402 self.state.save()?;
9403 return Ok(());
9404 }
9405 let wait = self.repository_wait();
9406 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9407 return Ok(());
9408 }
9409 let repo = self.state.repo.clone();
9410 let base = self.state.base_branch.clone();
9411 let mode = self.state.config.merge.mode;
9412 let style = self.state.config.merge.style;
9413 let facts = if is_review_run(&self.state) {
9414 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9415 let start = review_base(
9416 &repo,
9417 &self.state.config.merge.remote,
9418 &base,
9419 &self.state.base_commit,
9420 &winner.branch,
9421 )
9422 .await;
9423 branch_facts(&repo, &start, &winner.branch).await
9424 } else {
9425 None
9426 };
9427 let leaked = if is_review_run(&self.state) {
9430 leaked_subjects(&self.state, &winner.branch).await
9431 } else {
9432 Some(Vec::new())
9433 };
9434 let Some(pr) = self
9438 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9439 .await?
9440 else {
9441 return Ok(());
9442 };
9443 let message = pr.commit_message();
9444
9445 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9446 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9447 return Ok(());
9448 }
9449 let outcome = match mode {
9450 MergeMode::None => MergeOutcome {
9451 mode,
9452 ok: true,
9453 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9454 empty: false,
9455 },
9456 MergeMode::Pr | MergeMode::Local
9457 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9458 {
9459 MergeOutcome {
9460 mode,
9461 ok: false,
9462 detail: empty_candidate_detail(&self.state, &base),
9463 empty: true,
9464 }
9465 }
9466 MergeMode::Local => {
9467 let on = git::current_branch(&repo).await?;
9468 if on.as_deref() != Some(base.as_str()) {
9469 MergeOutcome {
9470 mode,
9471 ok: false,
9472 detail: format!(
9473 "{} has {} checked out, not the base branch {base}",
9474 repo.display(),
9475 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9476 ),
9477 empty: false,
9478 }
9479 } else if !git::is_clean(&repo).await? {
9480 MergeOutcome {
9481 mode,
9482 ok: false,
9483 detail: format!("{} is dirty; refusing to merge", repo.display()),
9484 empty: false,
9485 }
9486 } else {
9487 let out = match style {
9488 MergeStyle::Merge => {
9489 git::merge_no_ff(&repo, &winner.branch, &message).await?
9490 }
9491 MergeStyle::Squash => {
9492 git::merge_squash(&repo, &winner.branch, &message).await?
9493 }
9494 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9495 };
9496 MergeOutcome {
9497 mode,
9498 ok: out.ok(),
9499 detail: if out.ok() { out.stdout } else { out.stderr },
9500 empty: false,
9501 }
9502 }
9503 }
9504 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9505 self.state.pr_publication_pending = true;
9506 self.state.save()?;
9507 let out =
9508 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9509 self.state.pr_publication_pending = out.is_err();
9510 MergeOutcome {
9511 mode,
9512 ok: out.is_ok(),
9513 detail: match out {
9514 Ok(url) => url,
9515 Err(e) => format!("{e:#}"),
9516 },
9517 empty: false,
9518 }
9519 }
9520 MergeMode::Pr => {
9521 let remote = self.state.config.merge.remote.clone();
9522 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9523 let pushed = if self.state.foreign_checkout.is_some() {
9524 let refspec = format!("{delivery_head}:refs/heads/{public}");
9525 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9526 } else {
9527 git::push(&winner.worktree, &remote, &winner.branch).await?
9528 };
9529 if !pushed.ok() {
9530 MergeOutcome {
9531 mode,
9532 ok: false,
9533 detail: pushed.stderr,
9534 empty: false,
9535 }
9536 } else {
9537 if self.state.foreign_checkout.is_some() {
9538 self.state.pr_pushed_head = Some(delivery_head.clone());
9539 }
9540 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9546 let out = match pr_merge_plan(found) {
9547 PrPlan::Create => {
9548 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9549 .await
9550 }
9551 PrPlan::Adopt { url, title } => {
9552 self.state
9553 .event("merge", format!("Pr: adopted open pull request {url}"));
9554 if self.state.github_text.as_ref().is_some_and(|g| {
9555 g.resolved
9556 && g.choice == Some(GithubTextChoice::Rewrite)
9557 && g.rewrite.is_some()
9558 }) {
9559 land::set_pr_text_verbatim(
9562 &winner.worktree,
9563 &url,
9564 &pr.title,
9565 &pr.body,
9566 )
9567 .await
9568 .map(|()| url)
9569 } else {
9570 if title != pr.title
9571 && (!is_review_run(&self.state)
9572 || leaked
9573 .as_deref()
9574 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9575 && let Err(e) =
9576 if self.state.github_text.as_ref().is_some_and(|g| {
9577 g.resolved
9578 && g.title
9579 && g.choice == Some(GithubTextChoice::Original)
9580 }) {
9581 land::set_pr_title_verbatim(
9583 &winner.worktree,
9584 &url,
9585 &pr.title,
9586 )
9587 .await
9588 } else {
9589 land::set_pr_title(
9590 &mut self.state,
9591 &winner.worktree,
9592 &url,
9593 &pr.title,
9594 )
9595 .await
9596 }
9597 {
9598 tracing::warn!("could not refresh title of {url}: {e:#}");
9599 self.state
9600 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9601 }
9602 Ok(url)
9603 }
9604 }
9605 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9606 };
9607 match out {
9608 Ok(url) => MergeOutcome {
9609 mode,
9610 ok: true,
9611 detail: url,
9612 empty: false,
9613 },
9614 Err(e) => MergeOutcome {
9615 mode,
9616 ok: false,
9617 detail: e.to_string(),
9618 empty: false,
9619 },
9620 }
9621 }
9622 }
9623 };
9624
9625 self.state.status = match (mode, outcome.ok) {
9626 (MergeMode::None, _) => RunStatus::Ready,
9627 (_, true) if outcome.empty => RunStatus::Ready,
9628 (MergeMode::Pr, true) => RunStatus::Ready,
9629 (_, true) => RunStatus::Merged,
9630 (_, false) => RunStatus::Blocked,
9631 };
9632 self.state.event(
9633 "merge",
9634 format!(
9635 "{:?}: {}",
9636 mode,
9637 outcome.detail.lines().next().unwrap_or("")
9638 ),
9639 );
9640 self.state.merge = Some(outcome);
9641 self.state.save()?;
9642
9643 if self.state.config.graph.land
9649 && mode == MergeMode::Pr
9650 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9651 {
9652 self.run_land().await?;
9653 }
9654 self.settle_questions();
9659 Ok(())
9660 }
9661
9662 async fn run_land(&mut self) -> Result<()> {
9673 crate::pr_head::ensure_owner(&self.state)?;
9674 let url = self
9675 .state
9676 .pr
9677 .as_ref()
9678 .map(|pr| pr.url.clone())
9679 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9680 .unwrap_or_default();
9681 let url = url.lines().next().unwrap_or("").trim().to_owned();
9682 if !url.starts_with("http") {
9683 return Ok(());
9684 }
9685 loop {
9688 match land::land(&mut self.state, &url).await {
9689 Ok(_)
9690 if self
9691 .state
9692 .merge_repairs
9693 .last()
9694 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9695 {
9696 if self.repair_merge_review().await? {
9697 continue;
9698 }
9699 }
9700 Ok(_) if self.state.resource_block.is_some() => {
9701 return self.stop_for_resource();
9702 }
9703 Ok(pr) if self.state.parked => {
9704 let _ = pr;
9708 }
9709 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9710 self.state.status = RunStatus::Superseded;
9711 self.state.save()?;
9712 }
9713 Ok(pr) => {
9714 self.state.status = match pr.state {
9715 land::PrLifecycle::Merged => RunStatus::Merged,
9716 _ => RunStatus::Blocked,
9717 };
9718 if bump::should_release_bump(self.state.status)
9725 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9726 {
9727 self.state
9733 .event("bump", format!("release bump skipped: {e:#}"));
9734 }
9735 if self.state.status == RunStatus::Merged {
9738 crate::followup::after_merge(&mut self.state, &pr.url).await;
9739 }
9740 self.state.save()?;
9741 }
9742 Err(e) => {
9743 if e.is::<crate::repo_lock::ResourceBlocked>() {
9744 return Err(e);
9745 }
9746 self.state.status = RunStatus::Blocked;
9747 self.state.event("land", format!("gave up: {e}"));
9748 self.state.save()?;
9749 }
9750 }
9751 break;
9752 }
9753 Ok(())
9754 }
9755
9756 async fn repair_merge_review(&mut self) -> Result<bool> {
9759 use crate::run::MergeRepairPhase as Phase;
9760 crate::pr_head::ensure_owner(&self.state)?;
9761 let i = self.state.merge_repairs.len() - 1;
9762 let request = self.state.merge_repairs[i].clone();
9763 if request.phase == Phase::Fixing {
9764 let restored = match self.state.winner().cloned() {
9765 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9766 None => Err(anyhow::anyhow!("repair has no winning branch")),
9767 };
9768 let why = match restored {
9769 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9770 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9771 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9772 };
9773 self.hold_merge_repair(i, why)?;
9774 return Ok(false);
9775 }
9776 if request.phase == Phase::Reviewing && self.state.gate_ran {
9781 let winner = self
9782 .state
9783 .winner()
9784 .context("repair has no winning branch")?;
9785 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9786 if self.state.gate_head.as_deref() != Some(head.as_str())
9787 || !git::is_clean(&winner.worktree).await?
9788 {
9789 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9790 return Ok(false);
9791 }
9792 }
9793 let failure = if i >= self.state.config.graph.land_rounds
9794 || self.state.config.graph.review_rounds == 0
9795 {
9796 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9797 } else {
9798 None
9799 };
9800 if let Some(why) = failure {
9801 self.hold_merge_repair(i, why)?;
9802 return Ok(false);
9803 }
9804 if request.phase == Phase::Requested {
9805 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9806 Ok(pr) => pr,
9807 Err(e) => {
9808 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9809 return Ok(false);
9810 }
9811 };
9812 self.state.merge_repairs[i].phase = Phase::Fixing;
9813 self.state.land_approval = None;
9814 self.state.land_armed_head = None;
9815 self.state.save()?;
9816 let evidence = serde_json::to_string_pretty(&request)?;
9819 let reason = format!(
9820 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9821 );
9822 let budget = self.state.config.graph.land_rounds;
9823 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9824 match result {
9825 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9826 Ok(land::Fixed::Deferred) => {
9827 let worktree = self
9831 .state
9832 .winner()
9833 .context("repair has no winning branch")?
9834 .worktree
9835 .clone();
9836 self.graft_restore(&worktree, &request.head).await?;
9837 self.state.merge_repairs[i].phase = Phase::Requested;
9838 self.stop_for_resource()?;
9839 return Ok(false);
9840 }
9841 Ok(land::Fixed::Failed(why)) => {
9842 self.hold_merge_repair(
9843 i,
9844 format!(
9845 "repair failed: {why}; inspect the branch and run an explicit review"
9846 ),
9847 )?;
9848 return Ok(false);
9849 }
9850 Err(e) => {
9851 self.hold_merge_repair(
9852 i,
9853 format!(
9854 "repair failed: {e:#}; inspect the branch and run an explicit review"
9855 ),
9856 )?;
9857 return Ok(false);
9858 }
9859 }
9860 self.begin_merge_repair_review(i)?;
9864 }
9865 self.review_loop().await?;
9866 self.gate().await?;
9867 if self.resource_halt()? || self.state.parked {
9868 return Ok(false);
9869 }
9870 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9871 != Some(RunStatus::Gating)
9872 || !self.state.gate_allows_progress()
9873 {
9874 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9875 return Ok(false);
9876 }
9877 let winner = self
9880 .state
9881 .winner()
9882 .cloned()
9883 .context("repair has no winning branch")?;
9884 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9885 if self.state.gate_head.as_deref() != Some(after.as_str())
9886 || !git::is_clean(&winner.worktree).await?
9887 {
9888 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9889 return Ok(false);
9890 }
9891 let expected = self
9892 .state
9893 .pr_pushed_head
9894 .as_deref()
9895 .unwrap_or(&request.head)
9896 .to_owned();
9897 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9898 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9901 return Ok(false);
9902 }
9903 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9904 let refspec = format!("{after}:refs/heads/{public}");
9905 let push = git::git_raw(
9906 &winner.worktree,
9907 &["push", &lease, &self.state.config.merge.remote, &refspec],
9908 )
9909 .await;
9910 match push {
9911 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9912 Ok(out) => {
9913 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9914 return Ok(false);
9915 }
9916 Err(e) => {
9917 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9918 return Ok(false);
9919 }
9920 }
9921 self.state.merge_repairs[i].phase = Phase::Complete;
9922 self.state.status = RunStatus::Landing;
9923 self.state.event(
9924 "repair",
9925 "new review and gate completed; fresh owner merge decision required",
9926 );
9927 self.state.save()?;
9928 Ok(true)
9929 }
9930
9931 fn merge_repair_artifacts(&self) -> PathBuf {
9932 let artifacts = agent::artifacts_dir(&self.state.dir());
9933 match self
9934 .state
9935 .merge_repairs
9936 .last()
9937 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9938 {
9939 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9940 None => artifacts,
9941 }
9942 }
9943
9944 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9945 if let Some(e2e) = self.state.head_e2e.take() {
9946 self.state
9947 .verification_history
9948 .push(crate::run::VerificationEvidence {
9949 superseded_by: None,
9950 archived_at: Some(Timestamp::now()),
9951 fixes_spent: Some(self.state.gate_fixes.len()),
9952 gate_head: None,
9953 gate_ran: false,
9954 gate: Vec::new(),
9955 e2e: Some(e2e),
9956 });
9957 }
9958 self.state.reviews.clear();
9959 self.state.gate.clear();
9960 self.state.gate_ran = false;
9961 self.state.gate_head = None;
9962 self.state.gate_fixes.clear();
9963 self.state.gate_repair_e2e = None;
9964 self.state.pre_gate.clear();
9965 self.state.pre_gate_commit = None;
9966 self.state.contested_handoff = None;
9967 self.state.land_reruns = None;
9968 self.state.land_approval = None;
9969 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9970 self.state.status = RunStatus::Reviewing;
9971 self.state.save()
9972 }
9973
9974 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9975 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9976 self.state.merge_repairs[index].failure = Some(why.clone());
9977 self.state.status = RunStatus::Blocked;
9978 self.state.event("repair", why);
9979 self.state.save()
9980 }
9981
9982 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9986 if let Some(existing) = self.state.seats.get(key)
9987 && existing.agent == agent
9988 {
9989 return existing.clone();
9990 }
9991 let fresh = if self.state.seats.contains_key(key) {
9994 handover_seat(key, agent, self.state.next_seat_seed())
9995 } else {
9996 SeatState::new(key, agent, self.state.seed)
9997 };
9998 self.state.seats.insert(key.to_owned(), fresh.clone());
9999 fresh
10000 }
10001
10002 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10008 match self.state.seats.get(key) {
10009 Some(s) if s.agent != spec.id => {
10010 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10011 }
10012 _ => spec,
10013 }
10014 }
10015
10016 fn view(&self, c: &Candidate) -> CandidateView {
10018 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10019 .unwrap_or_default();
10020 let (patch, _) = blind::sanitize_patch(
10021 &format!("candidate {} patch", c.label),
10022 &raw,
10023 &self.state.config.blind,
10024 );
10025 CandidateView {
10026 label: c.label,
10027 branch: c.branch.clone(),
10028 summary: c.summary.clone(),
10029 stat: c.stat.clone(),
10030 patch,
10031 }
10032 }
10033
10034 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10036 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10037 prompt::judge(
10038 "(see above)",
10039 &views,
10040 self.roles.judges.len(),
10041 base_short,
10042 "en",
10043 )
10044 }
10045
10046 fn vote_prompt_full(
10052 &self,
10053 j: usize,
10054 labels: &[char],
10055 language: &str,
10056 candidates: &[Candidate],
10057 base_short: &str,
10058 ) -> String {
10059 let mut text = format!(
10060 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10061 prompt::final_vote(labels, language),
10062 self.state.instruction,
10063 self.candidate_block(candidates, base_short)
10064 );
10065 if let Some(own) = self
10066 .state
10067 .judgements
10068 .get(j)
10069 .filter(|r| !r.ranking.is_empty())
10070 {
10071 let reasons = own
10072 .reasons
10073 .iter()
10074 .map(|(k, v)| format!("- {k}: {v}"))
10075 .collect::<Vec<_>>()
10076 .join("\n");
10077 text.push_str(&format!(
10078 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10079 own.ranking.iter().collect::<String>(),
10080 if reasons.is_empty() {
10081 ""
10082 } else {
10083 ", because:\n"
10084 }
10085 ));
10086 }
10087 if !self.state.deliberation.is_empty() {
10088 text.push_str("\n# What was argued before this vote\n");
10089 for t in self.transcript(&[], j) {
10090 text.push_str(&format!(
10091 "\n## {}{}\n\n{}\n",
10092 t.who,
10093 if t.is_self { " (you)" } else { "" },
10094 t.body.trim()
10095 ));
10096 }
10097 }
10098 text
10099 }
10100
10101 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10108 let mut turns = Vec::new();
10109 for j in &self.state.judgements {
10110 if j.ranking.is_empty() {
10111 continue;
10112 }
10113 let reasons = j
10114 .reasons
10115 .iter()
10116 .map(|(k, v)| format!("- {k}: {v}"))
10117 .collect::<Vec<_>>()
10118 .join("\n");
10119 turns.push(Turn {
10120 who: format!("Judge {} (opening ranking)", j.judge),
10121 is_self: j.judge == self_idx + 1,
10122 body: format!(
10123 "Ranked {}{}{reasons}",
10124 j.ranking.iter().collect::<String>(),
10125 if reasons.is_empty() {
10126 ""
10127 } else {
10128 ", because:\n"
10129 }
10130 ),
10131 });
10132 }
10133 for t in self
10134 .state
10135 .deliberation
10136 .iter()
10137 .flat_map(|r| r.turns.iter())
10138 .chain(current)
10139 {
10140 turns.push(Turn {
10141 who: format!("Judge {}", t.judge),
10142 is_self: t.judge == self_idx + 1,
10143 body: t.body.clone(),
10144 });
10145 }
10146 turns
10147 }
10148}
10149
10150fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10152 agent::has_session(spec.kind, seat, sessions)
10153}
10154
10155fn next_untried_in_roster<'a>(
10176 roster: &'a [AgentSpec],
10177 start: usize,
10178 tried: &BTreeSet<String>,
10179) -> Option<&'a AgentSpec> {
10180 roster
10181 .get(start + 1..)?
10182 .iter()
10183 .find(|s| !tried.contains(&s.id))
10184}
10185
10186fn pick_successor<'a>(
10207 roster: &'a [AgentSpec],
10208 start: usize,
10209 tried: &BTreeSet<String>,
10210 carried: Option<&BTreeSet<String>>,
10211 others: &BTreeSet<String>,
10212) -> Option<&'a AgentSpec> {
10213 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10214 roster
10215 .get(start + 1..)
10216 .and_then(|tail| {
10217 tail.iter()
10218 .filter(free)
10219 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10220 })
10221 .or_else(|| {
10222 carried
10223 .is_some()
10224 .then(|| roster.iter().find(free))
10225 .flatten()
10226 })
10227 .or_else(|| match carried {
10228 Some(c) => next_for_seat(roster, start, tried, c),
10229 None => next_untried_in_roster(roster, start, tried),
10230 })
10231}
10232
10233fn next_for_seat<'a>(
10243 roster: &'a [AgentSpec],
10244 start: usize,
10245 round_tried: &BTreeSet<String>,
10246 carried_failed: &BTreeSet<String>,
10247) -> Option<&'a AgentSpec> {
10248 roster
10249 .get(start + 1..)?
10250 .iter()
10251 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10252 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10253}
10254
10255fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10261 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10262 return spec;
10263 };
10264 let ok = |id: &str| !h.failed.contains(id);
10265 if let Some(last) = h.last_ok.as_deref()
10266 && ok(last)
10267 && let Some(s) = roster.iter().find(|s| s.id == last)
10268 {
10269 return s.clone();
10270 }
10271 if ok(&spec.id) {
10272 return spec;
10273 }
10274 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10275 roster
10276 .get(from + 1..)
10277 .into_iter()
10278 .flatten()
10279 .chain(roster.iter())
10280 .find(|s| ok(&s.id))
10281 .cloned()
10282 .unwrap_or(spec)
10283}
10284
10285pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10289 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10290}
10291
10292const TIMED_OUT: &str = "timed out";
10295
10296impl FailClass {
10297 fn of(out: &AgentOutcome) -> Option<Self> {
10299 match out {
10300 AgentOutcome::Ok(_) => None,
10301 AgentOutcome::Quota(_) => Some(Self::Quota),
10302 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10303 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10304 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10305 AgentOutcome::ResourceBlocked(_) => None,
10308 }
10309 }
10310
10311 fn stem_word(&self) -> &'static str {
10313 match self {
10314 Self::Quota => "quota",
10315 _ => "handover",
10316 }
10317 }
10318}
10319
10320fn failure_signature(msg: &str) -> String {
10324 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10325 if let Some(code) = line
10328 .strip_prefix("process exited with no usable output (exit code ")
10329 .and_then(|s| s.strip_suffix(')'))
10330 .and_then(|s| s.parse::<i32>().ok())
10331 {
10332 line = format!("exited with some({code}) and no usable output");
10333 } else if matches!(
10334 line.as_str(),
10335 "process exited with no usable output (no exit code reported)"
10336 | "process exited with no usable output (exit code unavailable)"
10337 ) {
10338 line = "exited with none and no usable output".to_owned();
10339 }
10340 let mut out = Vec::new();
10341 for word in line.split_whitespace() {
10342 if word.contains('/') || word.contains('\\') {
10343 out.push("<path>".to_owned());
10344 continue;
10345 }
10346 let mut w = String::new();
10347 let mut in_digits = false;
10348 for c in word.chars() {
10349 if c.is_ascii_digit() {
10350 if !in_digits {
10351 w.push('#');
10352 }
10353 in_digits = true;
10354 } else {
10355 in_digits = false;
10356 w.push(c);
10357 }
10358 }
10359 out.push(w);
10360 }
10361 out.join(" ").chars().take(120).collect()
10362}
10363
10364fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10371 match cur {
10372 FailClass::Quota | FailClass::Timeout => true,
10373 FailClass::Other(_) => prev != Some(cur),
10374 }
10375}
10376
10377fn fail_reason(out: &AgentOutcome) -> String {
10379 match out {
10380 AgentOutcome::Ok(_) => String::new(),
10381 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10382 AgentOutcome::Dropped(o) => format!(
10383 "the CLI dropped the stream ({})",
10384 o.dropped
10385 .as_ref()
10386 .map(|d| d.why.as_str())
10387 .unwrap_or("it ended without delivering its answer")
10388 ),
10389 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10390 e.lines().next().unwrap_or("").chars().take(160).collect()
10391 }
10392 }
10393}
10394
10395pub(crate) fn record_handover(
10398 state: &mut RunState,
10399 node: &str,
10400 seat: &str,
10401 from: &str,
10402 to: &str,
10403 class: &FailClass,
10404 reason: &str,
10405) {
10406 let message = if *class == FailClass::Quota {
10407 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10408 } else {
10409 format!("{seat}: handed over {from} -> {to} ({reason})")
10410 };
10411 state.event(node, message);
10412 state.handovers.push(Handover {
10413 at: Timestamp::now(),
10414 node: node.to_owned(),
10415 seat: seat.to_owned(),
10416 from: from.to_owned(),
10417 to: to.to_owned(),
10418 reason: reason.to_owned(),
10419 });
10420}
10421
10422fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10436 commands.iter().any(|c| c.exit_code.is_none())
10437}
10438
10439fn verified_noop_claim(
10452 usable: bool,
10453 commands: &[agent::CommandEvidence],
10454 text: &str,
10455) -> Option<String> {
10456 (usable && !has_unconfirmed_command(commands))
10457 .then(|| verdict::verified_noop(text))
10458 .flatten()
10459}
10460
10461fn short(commit: &str) -> String {
10462 commit.chars().take(7).collect()
10463}
10464
10465fn make_executable(path: &Path) -> Result<()> {
10466 #[cfg(unix)]
10467 {
10468 use std::os::unix::fs::PermissionsExt as _;
10469 let mut perms = std::fs::metadata(path)?.permissions();
10470 perms.set_mode(0o755);
10471 std::fs::set_permissions(path, perms)?;
10472 }
10473 #[cfg(not(unix))]
10474 {
10475 let _ = path;
10476 }
10477 Ok(())
10478}
10479
10480struct WaveCtx<'a> {
10487 run: &'a str,
10490 node: &'a str,
10492 prompts: &'a Prompts,
10493 cache: Option<&'a Path>,
10495 round: Option<usize>,
10498 carry_seats: bool,
10502 pauses: Option<(&'a Pause, &'a Pause)>,
10505}
10506
10507fn repository_wait(
10509 state: &RunState,
10510 pauses: Option<(&Pause, &Pause)>,
10511) -> crate::repo_lock::WaitPolicy {
10512 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10513 let inherited = crate::repo_lock::WaitPolicy::current();
10514 crate::repo_lock::WaitPolicy::new(
10515 Duration::from_secs(state.config.graph.timeout_resource_wait),
10516 move || {
10517 inherited.cancelled()
10518 || pauses
10519 .as_ref()
10520 .is_some_and(|(a, b)| a.parked() || b.parked())
10521 },
10522 )
10523}
10524
10525fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10526 if error.is::<crate::repo_lock::ResourceBlocked>() {
10527 state.resource_block = Some(crate::run::ResourceBlock {
10528 cause: crate::run::ResourceBlockCause::Other,
10529 node: "repository".into(),
10530 kind: if error.to_string().contains("disk pressure") {
10531 "disk pressure"
10532 } else if error.to_string().contains("disk admission") {
10533 "disk admission"
10534 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10535 "incremental cache admission"
10536 } else {
10537 "repository coordination"
10538 }
10539 .into(),
10540 detail: error.to_string(),
10541 at: Timestamp::now(),
10542 });
10543 state.event("resource", error.to_string());
10544 }
10545}
10546
10547async fn coordinate_operation(
10548 state: &mut RunState,
10549 cwd: &Path,
10550 wait: &crate::repo_lock::WaitPolicy,
10551) -> Result<Option<crate::repo_lock::SeatLease>> {
10552 let result =
10553 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10554 .await;
10555 if let Err(e) = &result {
10556 record_repository_block(state, e);
10557 }
10558 result
10559}
10560
10561async fn run_one(
10563 job: SeatJob,
10564 sem: Arc<Semaphore>,
10565 ctx: &WaveCtx<'_>,
10566 state: &mut RunState,
10567 attempt: usize,
10568) -> (SeatState, AgentOutcome) {
10569 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10570 .await
10571 .pop()
10572 .expect("one job in, one result out");
10573 (seat, out)
10574}
10575
10576async fn wave(
10582 jobs: Vec<SeatJob>,
10583 sem: Arc<Semaphore>,
10584 ctx: &WaveCtx<'_>,
10585 state: &mut RunState,
10586 attempt: usize,
10587) -> Vec<(usize, SeatState, AgentOutcome)> {
10588 let WaveCtx {
10589 run,
10590 node,
10591 prompts,
10592 cache,
10593 round,
10594 carry_seats: _,
10595 pauses,
10596 } = *ctx;
10597 for job in &jobs {
10598 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10599 }
10600 if let Err(e) = state.save() {
10601 tracing::warn!("could not persist in-progress seats: {e:#}");
10606 }
10607 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10623 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10627 let cache_guard = if let Some(cache_dir) = cache {
10628 if jobs_had_a_writer {
10629 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10630 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10631 match acquire_cache_lease(
10632 state,
10633 cache_dir,
10634 &owner,
10635 resource_limit,
10636 node,
10637 node,
10638 &cancel,
10639 )
10640 .await
10641 {
10642 Ok(Some(g)) => Some(g),
10643 Ok(None) => {
10646 if let Some(block) = &mut state.resource_block {
10647 block.cause = crate::run::ResourceBlockCause::Cancelled {
10648 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10649 };
10650 }
10651 return block_wave(
10652 jobs,
10653 state,
10654 node,
10655 attempt,
10656 "parked while waiting for the shared build cache",
10657 );
10658 }
10659 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10662 Err(e) => {
10663 let why = format!("{e:#}");
10667 state.event(
10668 "resource",
10669 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10670 );
10671 return block_wave(jobs, state, node, attempt, &why);
10672 }
10673 }
10674 } else {
10675 None
10676 }
10677 } else {
10678 None
10679 };
10680 state.seats_spawning(node, attempt);
10683 if let Err(e) = state.save() {
10684 tracing::warn!("could not persist the start of a wave: {e:#}");
10685 }
10686 let slot_limit = resource_limit
10690 + jobs
10691 .iter()
10692 .map(|j| j.timeout)
10693 .max()
10694 .unwrap_or_default()
10695 .saturating_mul(jobs.len() as u32);
10696 let mut set = tokio::task::JoinSet::new();
10697 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10700 let overlay = prompts.overlay(node);
10701 let repository_wait = repository_wait(state, pauses);
10702 for (i, mut job) in jobs.into_iter().enumerate() {
10703 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10704 if cache.is_some() {
10705 job.prompt.push('\n');
10706 job.prompt
10707 .push_str(&prompt::build_cache_note(node, job.allow_write));
10708 }
10709 let sem = Arc::clone(&sem);
10710 let run = run.to_owned();
10711 let node = node.to_owned();
10712 let attachments = if node == "implement" {
10715 state.attachments.clone()
10716 } else {
10717 Vec::new()
10718 };
10719 let cache = cache
10730 .filter(|_| job.allow_write && cache_guard.is_some())
10731 .map(Path::to_path_buf);
10732 let spawned_tx = spawned_tx.clone();
10733 let repository_wait = repository_wait.clone();
10734 let coordination = state.config.coordination.clone();
10735 let fetched_base = state.fetched_base.clone();
10736 set.spawn(async move {
10737 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10741 Ok(p) => p,
10742 Err(_) => {
10743 return (
10744 i,
10745 job.seat,
10746 AgentOutcome::ResourceBlocked(format!(
10747 "no parallel slot became free within {}s",
10748 slot_limit.as_secs()
10749 )),
10750 false,
10751 );
10752 }
10753 };
10754 spawned_tx.send(job.seat.key.clone()).ok();
10755 let mut seat = job.seat;
10756 let out = repository_wait
10757 .scope(agent::invoke(
10758 &job.spec,
10759 &mut seat,
10760 &Invocation {
10761 coordination: coordination.as_ref(),
10762 fetched_base: fetched_base.as_ref(),
10763 cwd: &job.cwd,
10764 prompt: &job.prompt,
10765 timeout: job.timeout,
10766 allow_write: job.allow_write,
10767 sessions: job.sessions,
10768 artifacts: &job.artifacts,
10769 stem: &job.stem,
10770 run: &run,
10771 node: &node,
10772 cache_dir: cache.as_deref(),
10773 attachments: &attachments,
10774 writable: &[],
10775 },
10776 ))
10777 .await;
10778 let interrupted = out
10779 .as_ref()
10780 .err()
10781 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10782 let out = match out {
10783 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10784 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10785 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10793 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10794 Ok(o) => AgentOutcome::Failed(format!(
10795 "process exited with no usable output ({})",
10796 crate::notices::exit_status(o.exit_code)
10797 )),
10798 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10799 AgentOutcome::ResourceBlocked(e.to_string())
10800 }
10801 Err(e) => AgentOutcome::Failed(e.to_string()),
10802 };
10803 (i, seat, out, interrupted)
10804 });
10805 }
10806 drop(spawned_tx);
10807 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10808 loop {
10809 let joined = tokio::select! {
10810 Some(key) = spawned_rx.recv() => {
10811 state.seat_spawned(&key);
10813 if let Err(e) = state.save() {
10814 tracing::warn!("could not persist a seat's start: {e:#}");
10815 }
10816 continue;
10817 }
10818 joined = set.join_next() => joined,
10819 };
10820 let Some(joined) = joined else {
10821 break;
10822 };
10823 let (i, seat, out, interrupted) = match joined {
10824 Ok(v) => v,
10825 Err(e) => {
10829 tracing::error!("agent task panicked: {e}");
10830 continue;
10831 }
10832 };
10833 state.seat_finished(&seat.key);
10834 if interrupted
10835 && node == "implement"
10836 && let Some(label) = state
10837 .candidates
10838 .iter()
10839 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10840 .map(|candidate| candidate.label)
10841 {
10842 state.candidate_preservation.insert(
10843 label,
10844 "interrupted implementation awaiting persistence".into(),
10845 );
10846 state.interrupted_implementations.insert(label);
10847 }
10848 if let AgentOutcome::ResourceBlocked(why) = &out
10849 && state.resource_block.is_none()
10850 {
10851 state.resource_block = Some(crate::run::ResourceBlock {
10852 cause: crate::run::ResourceBlockCause::Other,
10853 node: node.to_owned(),
10854 kind: if why.contains("disk pressure") {
10855 "disk pressure"
10856 } else if why.contains("disk admission") {
10857 "disk admission"
10858 } else if why.contains(crate::incremental::BLOCKED) {
10859 "incremental cache admission"
10860 } else if why.contains("repository coordination") {
10861 "repository coordination"
10862 } else {
10863 "parallel slot"
10864 }
10865 .to_owned(),
10866 detail: why.clone(),
10867 at: Timestamp::now(),
10868 });
10869 let activity = if interrupted {
10870 "agent execution interrupted"
10871 } else {
10872 "no agent was started"
10873 };
10874 state.event("resource", format!("{node}: {activity} - {why}"));
10875 }
10876 record_jobs(state, node, round, &seat.key, &out);
10877 if let Err(e) = state.save() {
10878 tracing::warn!("could not persist a seat's completion: {e:#}");
10879 }
10880 if collected.len() <= i {
10881 collected.resize_with(i + 1, || None);
10882 }
10883 collected[i] = Some((i, seat, out));
10884 }
10885 if state
10891 .active
10892 .values()
10893 .any(|a| a.node == node && a.attempt == attempt)
10894 {
10895 state
10896 .active
10897 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10898 if let Err(e) = state.save() {
10899 tracing::warn!("could not persist the end of a wave: {e:#}");
10900 }
10901 }
10902 if let Some(cache_dir) = cache
10909 && jobs_had_a_writer
10910 {
10911 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10912 }
10913 if let Some(guard) = cache_guard {
10914 guard.release();
10915 }
10916 collected.into_iter().flatten().collect()
10917}
10918
10919fn record_jobs(
10930 state: &mut RunState,
10931 node: &str,
10932 round: Option<usize>,
10933 seat: &str,
10934 out: &AgentOutcome,
10935) {
10936 let commands: &[agent::CommandEvidence] = match out {
10937 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10938 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10939 };
10940 let checked_at = Timestamp::now();
10941 for c in commands {
10942 state.jobs.push(JobRecord {
10943 node: node.to_owned(),
10944 round,
10945 seat: seat.to_owned(),
10946 id: c.id.clone(),
10947 description: c.description.clone(),
10948 checked_at,
10949 status: match c.exit_code {
10950 Some(0) => JobStatus::Completed,
10951 Some(_) => JobStatus::Failed,
10952 None => JobStatus::Unknown,
10953 },
10954 exit_code: c.exit_code,
10955 result_summary: c.result_summary.clone(),
10956 source: c.source.clone(),
10957 });
10958 }
10959}
10960
10961fn round_is_clean(
10982 blocking: usize,
10983 e2e_ok: bool,
10984 answered: usize,
10985 expected: usize,
10986 quota_missing: usize,
10987 policy: IncompleteReviewPolicy,
10988) -> bool {
10989 if blocking != 0 || !e2e_ok {
10990 return false;
10991 }
10992 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10993 return true;
10994 }
10995 answered > 0 && expected - answered <= quota_missing
10996}
10997
10998pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11022 if max_rounds == 0
11023 || reviews.last().is_some_and(|r| {
11024 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11025 })
11026 {
11027 return Some(RunStatus::Gating);
11028 }
11029 let last = reviews.last()?;
11030 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11031 if reviews.len() < max_rounds && !stagnant {
11032 return None;
11033 }
11034 if last.incomplete() && last.blocking == 0 {
11035 return Some(RunStatus::Blocked);
11036 }
11037 if last.e2e_status() == E2eStatus::ResourceBlocked
11038 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11039 {
11040 return None;
11041 }
11042 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11043 RunStatus::Gating
11044 } else {
11045 RunStatus::Blocked
11046 })
11047}
11048
11049pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11064 if nudged {
11065 (full / 4).max(Duration::from_secs(120)).min(full)
11066 } else {
11067 full
11068 }
11069}
11070
11071#[allow(clippy::too_many_arguments)]
11100async fn ask_json_wave<T>(
11101 jobs: Vec<SeatJob>,
11102 sem: Arc<Semaphore>,
11103 retries: usize,
11104 roster: &[AgentSpec],
11105 ctx: &WaveCtx<'_>,
11106 losses: &mut Vec<QuotaLoss>,
11107 state: &mut RunState,
11108 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11109) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11110where
11111 T: serde::de::DeserializeOwned + Send + 'static,
11112{
11113 ask_wave_with(
11114 jobs,
11115 sem,
11116 retries,
11117 roster,
11118 ctx,
11119 losses,
11120 state,
11121 &|text: &str| {
11122 let v = verdict::extract_json::<T>(text)?;
11123 validate(&v)?;
11124 Ok(v)
11125 },
11126 )
11127 .await
11128}
11129
11130#[allow(clippy::too_many_arguments)]
11139async fn ask_wave_with<T>(
11140 jobs: Vec<SeatJob>,
11141 sem: Arc<Semaphore>,
11142 retries: usize,
11143 roster: &[AgentSpec],
11144 ctx: &WaveCtx<'_>,
11145 losses: &mut Vec<QuotaLoss>,
11146 state: &mut RunState,
11147 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11148) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11149where
11150 T: Send + 'static,
11151{
11152 let n = jobs.len();
11153 let originals: Vec<SeatJob> = jobs;
11154 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11155 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11156 let mut nudges: Vec<usize> = vec![0; n];
11163 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11167 let mut tried: Vec<BTreeSet<String>> = specs
11168 .iter()
11169 .map(|s| BTreeSet::from([s.id.clone()]))
11170 .collect();
11171 let starts: Vec<usize> = specs
11172 .iter()
11173 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11174 .collect();
11175 let carry = ctx.carry_seats && !roster.is_empty();
11176 let carried: Vec<BTreeSet<String>> = originals
11179 .iter()
11180 .map(|j| {
11181 state
11182 .seat_history
11183 .get(&j.seat.key)
11184 .filter(|_| carry)
11185 .map(|h| h.failed.clone())
11186 .unwrap_or_default()
11187 })
11188 .collect();
11189 let mut prev: Vec<Option<FailClass>> = originals
11190 .iter()
11191 .map(|j| {
11192 state
11193 .seat_history
11194 .get(&j.seat.key)
11195 .filter(|_| carry)
11196 .and_then(|h| h.last_fail.clone())
11197 })
11198 .collect();
11199 let next_agent =
11200 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11201 let others: BTreeSet<String> = specs
11202 .iter()
11203 .enumerate()
11204 .filter(|(j, _)| *j != i)
11205 .map(|(_, s)| s.id.clone())
11206 .collect();
11207 pick_successor(
11208 roster,
11209 starts[i],
11210 tried,
11211 carry.then(|| &carried[i]),
11212 &others,
11213 )
11214 .cloned()
11215 };
11216 let mut fresh: Vec<Option<String>> = vec![None; n];
11217 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11218 let mut pending: Vec<usize> = (0..n).collect();
11219
11220 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11225 for round in 0..max_rounds {
11226 if pending.is_empty() {
11227 break;
11228 }
11229 let mut batch = Vec::with_capacity(pending.len());
11230 let mut renudged: Vec<&str> = Vec::new();
11231 for &i in &pending {
11232 let src = &originals[i];
11233 let full: &str = match &src.handover {
11238 Some(h) if specs[i].id != src.spec.id => h,
11239 _ => &src.prompt,
11240 };
11241 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11244 (
11245 full.to_owned(),
11246 src.timeout,
11247 format!("{}-{word}-{}", src.stem, specs[i].id),
11248 )
11249 } else if nudges[i] == 0 {
11250 (full.to_owned(), src.timeout, src.stem.clone())
11251 } else {
11252 renudged.push(src.seat.key.as_str());
11253 let why = done[i]
11254 .as_ref()
11255 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11256 .unwrap_or_else(|| "no parsable answer".to_owned());
11257 let nudge = prompt::nudge(&why);
11258 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11259 let prompt = if nudged {
11260 nudge
11261 } else {
11262 format!("{full}\n\n---\n\n{nudge}")
11263 };
11264 (
11265 prompt,
11266 retry_budget(src.timeout, nudged),
11267 format!("{}-retry{}", src.stem, nudges[i]),
11268 )
11269 };
11270 batch.push(SeatJob {
11271 spec: specs[i].clone(),
11272 seat: seats[i].clone(),
11273 cwd: src.cwd.clone(),
11274 prompt,
11275 timeout,
11276 allow_write: src.allow_write,
11277 sessions: src.sessions,
11278 artifacts: src.artifacts.clone(),
11279 stem,
11280 handover: None,
11281 });
11282 }
11283
11284 if !renudged.is_empty() {
11285 state.event(
11286 ctx.node,
11287 format!("retry {round}: re-asking {}", renudged.join(", ")),
11288 );
11289 }
11290 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11291 let mut still = Vec::new();
11292 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11293 seats[i] = seat;
11294 if let AgentOutcome::ResourceBlocked(why) = &out {
11297 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11298 continue;
11299 }
11300 let class = FailClass::of(&out);
11301 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11304 && nudges[i] < retries
11305 && !roster.is_empty();
11306 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11307 let next = should_hand_over(prev[i].as_ref(), &cur)
11308 .then(|| next_agent(i, &tried[i], &specs))
11309 .flatten();
11310 if carry {
11311 let h = state
11312 .seat_history
11313 .entry(originals[i].seat.key.clone())
11314 .or_default();
11315 h.failed.insert(specs[i].id.clone());
11316 h.last_fail = Some(cur.clone());
11317 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11318 h.last_ok = None;
11319 }
11320 if let Err(e) = state.save() {
11323 tracing::warn!("could not persist a seat's failure history: {e:#}");
11324 }
11325 }
11326 if let Some(next) = next {
11327 record_handover(
11328 state,
11329 ctx.node,
11330 &originals[i].seat.key,
11331 &specs[i].id,
11332 &next.id,
11333 &cur,
11334 &fail_reason(&out),
11335 );
11336 tried[i].insert(next.id.clone());
11337 prev[i] = Some(cur.clone());
11338 seats[i] =
11339 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11340 specs[i] = next;
11341 fresh[i] = Some(cur.stem_word().to_owned());
11342 nudges[i] = 0;
11343 done[i] = Some(Err(anyhow::anyhow!(
11344 "handed over after: {}",
11345 fail_reason(&out)
11346 )));
11347 still.push(i);
11348 continue;
11349 }
11350 }
11351 if carry
11352 && !nudge_first
11353 && let Some(cur) = class.clone()
11354 {
11355 let h = state
11357 .seat_history
11358 .entry(originals[i].seat.key.clone())
11359 .or_default();
11360 h.failed.insert(specs[i].id.clone());
11361 h.last_fail = Some(cur);
11362 }
11363 let parsed = match out {
11364 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11365 AgentOutcome::Quota(o) => {
11366 last_quota[i] = o.quota.clone();
11367 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11368 }
11369 AgentOutcome::Dropped(o) => {
11373 let why = o
11374 .dropped
11375 .as_ref()
11376 .map(|d| d.why.as_str())
11377 .unwrap_or("the CLI ended the stream without delivering its answer");
11378 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11379 }
11380 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11381 Err(anyhow::anyhow!(e))
11382 }
11383 };
11384 let quota = class == Some(FailClass::Quota);
11385 let failed = parsed.is_err();
11386 if !failed && matches!(ctx.node, "judge" | "vote") {
11387 state
11390 .quota
11391 .retain(|loss| loss.seat != originals[i].seat.key);
11392 }
11393 done[i] = Some(parsed);
11394 if carry && !failed {
11395 let h = state
11396 .seat_history
11397 .entry(originals[i].seat.key.clone())
11398 .or_default();
11399 h.failed.remove(&specs[i].id);
11400 h.last_ok = Some(specs[i].id.clone());
11401 h.last_fail = None;
11402 }
11403 let agent_failure = class.is_some()
11411 && !nudge_first
11412 && !roster.is_empty()
11413 && next_agent(i, &tried[i], &specs).is_some();
11414 if failed && !quota && !agent_failure && nudges[i] < retries {
11415 nudges[i] += 1;
11416 still.push(i);
11417 }
11418 }
11419 pending = still;
11420 }
11421
11422 for (i, q) in last_quota.into_iter().enumerate() {
11425 if let Some(quota) = q {
11426 losses.push(QuotaLoss::from_report(
11427 originals[i].seat.key.clone(),
11428 ctx.node,
11429 Some("a),
11430 ));
11431 }
11432 }
11433
11434 if state.status != RunStatus::Stalled
11437 && ctx.node != "deliberate"
11438 && n > 0
11439 && losses.len() == n
11440 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11441 {
11442 block_machine_outage(state, ctx.node, losses);
11443 state.quota.append(losses);
11444 }
11445
11446 seats
11447 .into_iter()
11448 .zip(done)
11449 .zip(nudges)
11450 .map(|((seat, res), attempts)| {
11451 (
11452 seat,
11453 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11454 attempts,
11455 )
11456 })
11457 .collect()
11458}
11459
11460pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11463 let mut auth: Vec<_> = losses
11464 .iter()
11465 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11466 .map(|loss| loss.agent.clone())
11467 .collect();
11468 auth.sort();
11469 auth.dedup();
11470 let (kind, detail) = if auth.is_empty() {
11471 (
11472 "agent usage limit",
11473 "All seats reached their agent usage limits; resume after usage is available"
11474 .to_owned(),
11475 )
11476 } else {
11477 (
11478 "agent authentication",
11479 format!(
11480 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11481 auth.join(", ")
11482 ),
11483 )
11484 };
11485 state.resource_block = Some(crate::run::ResourceBlock {
11486 node: node.to_owned(),
11487 kind: kind.to_owned(),
11488 cause: crate::run::ResourceBlockCause::Other,
11489 detail: detail.clone(),
11490 at: Timestamp::now(),
11491 });
11492 state.event(node, detail);
11493}
11494
11495fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11497 results
11498 .iter()
11499 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11500}
11501
11502fn block_wave(
11506 jobs: Vec<SeatJob>,
11507 state: &mut RunState,
11508 node: &str,
11509 attempt: usize,
11510 why: &str,
11511) -> Vec<(usize, SeatState, AgentOutcome)> {
11512 state.resource_wait = None;
11513 state
11514 .active
11515 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11516 if let Err(e) = state.save() {
11517 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11518 }
11519 jobs.into_iter()
11520 .enumerate()
11521 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11522 .collect()
11523}
11524
11525async fn acquire_cache_lease(
11538 state: &mut RunState,
11539 cache_dir: &Path,
11540 owner: &crate::cache::Owner,
11541 budget: Duration,
11542 node: &str,
11543 context: &str,
11544 cancel: &(dyn Fn() -> bool + Sync),
11545) -> Result<Option<crate::cache::Guard>> {
11546 let home = crate::run::home();
11547 let started = Instant::now();
11548 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11549 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11550 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11551 Err(e) => {
11552 state.event(
11553 "verify",
11554 format!("{context}: could not check the shared build cache: {e:#}"),
11555 );
11556 if let Err(e2) = state.save() {
11557 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11558 }
11559 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11560 }
11561 };
11562 state.event(
11563 "verify",
11564 format!(
11565 "{context}: waiting for the shared build cache at {} ({})",
11566 cache_dir.display(),
11567 busy.describe()
11568 ),
11569 );
11570 state.resource_wait = Some(crate::run::ResourceWait {
11573 node: context.to_owned(),
11574 kind: "shared build cache".to_owned(),
11575 holder: busy.describe(),
11576 since: Timestamp::now(),
11577 limit_secs: budget.as_secs(),
11578 });
11579 if let Err(e) = state.save() {
11580 tracing::warn!("could not persist a cache wait: {e:#}");
11581 }
11582 let remaining = budget.saturating_sub(started.elapsed());
11583 let waited = crate::cache::wait_for_unless(
11584 &home,
11585 cache_dir,
11586 owner,
11587 remaining,
11588 Duration::from_secs(5),
11589 cancel,
11590 )
11591 .await;
11592 state.resource_wait = None;
11593 let cause = match &waited {
11594 Ok(Some(_)) => None,
11595 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11596 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11597 Some(crate::run::ResourceBlockCause::Timeout {
11598 waited_secs: started.elapsed().as_secs(),
11599 limit_secs: budget.as_secs(),
11600 })
11601 }
11602 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11603 };
11604 if let Some(cause) = cause {
11605 let detail = match &waited {
11606 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11607 Err(error) => format!("{error:#}"),
11608 };
11609 state.resource_block = Some(crate::run::ResourceBlock {
11610 cause,
11611 node: node.to_owned(),
11612 kind: "shared build cache".into(),
11613 detail,
11614 at: Timestamp::now(),
11615 });
11616 state.event(
11617 "resource",
11618 state.resource_block.as_ref().unwrap().description(),
11619 );
11620 }
11621 if let Err(error) = state.save() {
11622 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11623 }
11624 waited
11625}
11626
11627#[derive(Debug)]
11632struct CacheCheckError(String);
11633
11634impl std::fmt::Display for CacheCheckError {
11635 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11636 f.write_str(&self.0)
11637 }
11638}
11639
11640impl std::error::Error for CacheCheckError {}
11641
11642fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11644 state
11645 .resource_block
11646 .get_or_insert_with(|| crate::run::ResourceBlock {
11647 cause: crate::run::ResourceBlockCause::Other,
11648 node: node.to_owned(),
11649 kind: "shared build cache".into(),
11650 detail: format!("{error:#}"),
11651 at: Timestamp::now(),
11652 });
11653 if let Err(error) = state.save() {
11654 tracing::warn!("could not persist cache resource block: {error:#}");
11655 }
11656}
11657
11658#[allow(clippy::too_many_arguments)]
11679async fn with_cache_lease<'s, F, Fut>(
11680 state: &'s mut RunState,
11681 cache_dir: Option<&Path>,
11682 node: &str,
11683 seat: &str,
11684 worktree: &Path,
11685 head: &str,
11686 budget: Duration,
11687 context: &str,
11688 wait: &crate::repo_lock::WaitPolicy,
11689 body: F,
11690) -> Result<(Vec<CommandOutcome>, bool)>
11691where
11692 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11693 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11694{
11695 let started = Instant::now();
11698 let home = crate::run::home();
11699 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11700 let guard = if let Some(cache_dir) = cache_dir {
11701 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11702 match acquire_cache_lease(
11703 state,
11704 cache_dir,
11705 &owner,
11706 resource_limit,
11707 node,
11708 context,
11709 &|| wait.cancelled(),
11710 )
11711 .await
11712 .and_then(|g| {
11713 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11714 }) {
11715 Ok(g) => Some(g),
11716 Err(e) => {
11717 record_cache_error(state, node, &e);
11718 return Ok((
11719 vec![CommandOutcome {
11720 command: "(waiting for the shared build cache)".into(),
11721 code: None,
11722 output_tail: e.to_string(),
11723 duration_ms: started.elapsed().as_millis() as u64,
11724 resource_blocked: true,
11725 }],
11726 false,
11727 ));
11728 }
11729 }
11730 } else {
11731 None
11732 };
11733 let _coordination = coordinate_operation(state, worktree, wait).await?;
11734 if let Some(cache_dir) = cache_dir {
11735 let identity = crate::cache::Identity::new(worktree, head);
11736 if let Err(e) = crate::cache::ensure_fresh(
11737 &home,
11738 cache_dir,
11739 &identity,
11740 &state.config.verify.cargo_manifests(),
11741 ) {
11742 record_cache_error(state, node, &e);
11743 state.event(
11744 "verify",
11745 format!(
11746 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11747 worktree.display(),
11748 short(head)
11749 ),
11750 );
11751 if let Some(guard) = guard {
11752 guard.release();
11753 }
11754 return Ok((
11755 vec![CommandOutcome {
11756 command: "(confirming the shared build cache is fresh)".into(),
11757 code: None,
11758 output_tail: e.to_string(),
11759 duration_ms: started.elapsed().as_millis() as u64,
11760 resource_blocked: true,
11761 }],
11762 false,
11763 ));
11764 }
11765 }
11766 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11767 if !timed_out_pids.is_empty() {
11768 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11769 }
11770 drop(_coordination);
11773 if let Some(guard) = guard {
11774 guard.release();
11775 }
11776 Ok((outcomes, retried))
11777}
11778
11779async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11791 wait_for_pids_with(
11792 pids,
11793 crate::proc::pid_alive,
11794 LEASE_RELEASE_POLL,
11795 LEASE_RELEASE_MAX_WAIT,
11796 )
11797 .await;
11798}
11799
11800async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11806 pids: &[u32],
11807 alive: F,
11808 poll: Duration,
11809 max_wait: Duration,
11810) {
11811 let deadline = Instant::now() + max_wait;
11812 loop {
11813 if pids.iter().all(|&pid| !alive(pid)) {
11814 return;
11815 }
11816 if Instant::now() >= deadline {
11817 return;
11818 }
11819 tokio::time::sleep(poll).await;
11820 }
11821}
11822
11823fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11830 outcomes.iter().any(|o| o.resource_blocked)
11831}
11832
11833pub(crate) enum GateFix {
11835 Retry,
11837 Stop,
11840 Defer,
11843}
11844
11845pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11853 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11854 red.peek().is_some()
11855 && red.all(|o| {
11856 !o.resource_blocked
11857 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11858 && !o.output_tail.trim().is_empty()
11859 })
11860}
11861
11862fn e2e_outcome_label(o: &CommandOutcome) -> String {
11866 if o.ok() {
11867 return "pass".to_owned();
11868 }
11869 let reason = if o.build_failed() {
11870 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11871 } else {
11872 format!("FAIL ({:?})", o.code)
11873 };
11874 format!(
11875 "{reason}\n{}",
11876 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11877 )
11878}
11879
11880async fn run_e2e_with_retry(
11888 state: &mut RunState,
11889 shell: &[String],
11890 commands: &[String],
11891 worktree: &Path,
11892 timeout: Duration,
11893 context: &str,
11894) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11895 let (mut e2e, mut timed_out_pids) = run_commands(
11896 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11897 )
11898 .await;
11899 for o in &e2e {
11900 state.event(
11901 "verify",
11902 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11903 );
11904 }
11905 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11909 if verify_retried {
11910 state.event(
11911 "verify",
11912 format!(
11913 "{context}: verify could not build/link, not a test result — retrying once \
11914 before concluding"
11915 ),
11916 );
11917 let retried = run_commands(
11918 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11919 )
11920 .await;
11921 e2e = retried.0;
11922 timed_out_pids.extend(retried.1);
11925 for o in &e2e {
11926 state.event(
11927 "verify",
11928 format!(
11929 "{context}: retry `{}` -> {}",
11930 o.command,
11931 e2e_outcome_label(o)
11932 ),
11933 );
11934 }
11935 }
11936 (e2e, verify_retried, timed_out_pids)
11937}
11938
11939#[allow(clippy::too_many_arguments)]
11955async fn run_commands(
11956 state: &mut RunState,
11957 node: &str,
11958 task: &str,
11959 attempt: usize,
11960 shell: &[String],
11961 commands: &[String],
11962 cwd: &Path,
11963 timeout: Duration,
11964) -> (Vec<CommandOutcome>, Vec<u32>) {
11965 if commands.is_empty() {
11966 return (Vec::new(), Vec::new());
11971 }
11972 let mut out = Vec::new();
11973 let mut timed_out_pids = Vec::new();
11974 let total = commands.len();
11975 for (idx, command) in commands.iter().enumerate() {
11976 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11977 if let Err(e) = state.save() {
11978 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11979 }
11980 let started = Instant::now();
11981 let mut cmd = tokio::process::Command::new(&shell[0]);
11982 cmd.quiet();
11983 for key in crate::worker::INVOCATION_ENV {
11987 cmd.env_remove(key);
11988 }
11989 cmd.args(&shell[1..])
11990 .arg(command)
11991 .current_dir(cwd)
11992 .stdin(std::process::Stdio::null())
11993 .stdout(std::process::Stdio::piped())
11994 .stderr(std::process::Stdio::piped())
11995 .kill_on_drop(true);
11996 let mut outputs = Vec::new();
11997 let protected = (|| -> Result<()> {
11998 crate::incremental::configure(&mut cmd, cwd)?;
11999 let mut paths = vec![cwd.to_owned()];
12000 if let Some(cache) = state.config.cache_dir() {
12001 paths.push(cache);
12002 }
12003 outputs.extend(crate::retention::OutputLease::admit(
12004 &crate::run::home(),
12005 &paths,
12006 state.config.disk.min_free_bytes,
12007 )?);
12008 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12009 outputs.push(guard);
12010 }
12011 if let Some(cache) = state.config.cache_dir() {
12012 outputs.push(crate::retention::OutputLease::cache(
12013 &crate::run::home(),
12014 &cache,
12015 false,
12016 )?);
12017 }
12018 Ok(())
12019 })();
12020 if let Err(e) = protected {
12021 let detail = format!("execution admission unavailable: {e:#}");
12022 state.resource_block = Some(crate::run::ResourceBlock {
12023 cause: crate::run::ResourceBlockCause::Other,
12024 node: node.to_owned(),
12025 kind: if detail.contains("disk admission") {
12026 "disk admission"
12027 } else {
12028 "output ownership"
12029 }
12030 .to_owned(),
12031 detail: detail.clone(),
12032 at: Timestamp::now(),
12033 });
12034 out.push(CommandOutcome {
12035 command: command.clone(),
12036 code: None,
12037 output_tail: detail,
12038 duration_ms: 0,
12039 resource_blocked: true,
12040 });
12041 break;
12042 }
12043 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12044 let (code, body) = match spawned {
12045 Ok(mut running) => {
12046 let pid = running.child().id();
12047 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12048 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12049 let mut paths = vec![cwd.to_owned()];
12050 if let Some(cache) = state.config.cache_dir() {
12051 paths.push(cache);
12052 }
12053 let mut pressure = None;
12054 let status = tokio::select! {
12055 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12056 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12057 pressure = Some(reason); None
12058 }
12059 };
12060 let (code, failure) = match status {
12061 Some(Ok(Ok(status))) => (
12062 status.code(),
12063 status
12064 .code()
12065 .is_none()
12066 .then(|| format!("command terminated: {status}")),
12067 ),
12068 Some(Ok(Err(error))) => {
12069 (None, Some(format!("failed waiting for command: {error}")))
12070 }
12071 Some(Err(_)) => {
12072 if !running.coordinated()
12076 && let Some(pid) = pid
12077 {
12078 timed_out_pids.push(pid);
12079 }
12080 (
12081 None,
12082 Some(format!("timed out after {}s", timeout.as_secs())),
12083 )
12084 }
12085 None => (None, None),
12086 };
12087 running.finish().await;
12088 if let Some(reason) = pressure {
12089 state.resource_block = Some(crate::run::ResourceBlock {
12090 cause: crate::run::ResourceBlockCause::Other,
12091 node: node.to_owned(),
12092 kind: "disk pressure".to_owned(),
12093 detail: reason.clone(),
12094 at: Timestamp::now(),
12095 });
12096 out.push(CommandOutcome {
12097 command: command.clone(),
12098 code: None,
12099 output_tail: reason,
12100 duration_ms: started.elapsed().as_millis() as u64,
12101 resource_blocked: true,
12102 });
12103 break;
12104 }
12105 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12106 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12107 if let Some(failure) = failure {
12108 body.push('\n');
12109 body.push_str(&failure);
12110 }
12111 (code, body)
12112 }
12113 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12114 };
12115 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12116 if !body.is_empty() {
12117 match crate::run::write_verification_artifact(state, task, &body) {
12121 Ok(name) if summary != body => {
12122 summary = format!("[full output: artifacts/{name}]\n{summary}");
12123 }
12124 Ok(_) => {}
12125 Err(e) => {
12126 let warning = format!("could not save the full {task} output: {e:#}");
12127 tracing::warn!("{warning}");
12128 state.event(node, warning);
12129 if summary != body {
12130 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12131 }
12132 }
12133 }
12134 }
12135 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12136 out.push(CommandOutcome {
12137 command: command.clone(),
12138 code,
12139 output_tail: summary,
12140 duration_ms: started.elapsed().as_millis() as u64,
12141 resource_blocked,
12142 });
12143 if resource_blocked {
12144 state.resource_block = Some(crate::run::ResourceBlock {
12145 cause: crate::run::ResourceBlockCause::Other,
12146 node: node.to_owned(),
12147 kind: "incremental cache admission".into(),
12148 detail: body,
12149 at: Timestamp::now(),
12150 });
12151 break;
12152 }
12153 }
12154 state.task_finished(task);
12155 if let Err(e) = state.save() {
12156 tracing::warn!("could not persist the end of {task}: {e:#}");
12157 }
12158 (out, timed_out_pids)
12159}
12160
12161fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12173 let repo = repo.display();
12174 match style {
12175 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12176 MergeStyle::Squash => {
12177 let subject = message
12180 .lines()
12181 .next()
12182 .unwrap_or(branch)
12183 .replace(['\\', '"', '$', '`'], "");
12184 format!(
12185 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12186 )
12187 }
12188 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12189 }
12190}
12191
12192const PR_TITLE_MAX: usize = 240;
12203
12204fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12215 let line = queue::first_line(text)?;
12216 let chars: Vec<char> = line.chars().collect();
12217 let (kept, mark) = if chars.len() <= max {
12218 (line.to_owned(), "")
12219 } else if let Some(end) = sentence_end(&chars, max) {
12220 (chars[..end].iter().collect::<String>(), "")
12221 } else {
12222 let room = max.saturating_sub(3);
12223 let cut = if chars[room].is_whitespace() {
12226 room
12227 } else {
12228 chars[..room]
12229 .iter()
12230 .rposition(|c| c.is_whitespace())
12231 .unwrap_or(room)
12232 };
12233 let head: String = chars[..cut].iter().collect();
12234 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12235 (head.to_owned(), "...")
12236 };
12237 if kept.is_empty() {
12238 return None;
12239 }
12240 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12241 return None;
12242 }
12243 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12244 return None;
12245 }
12246 Some(format!("{kept}{mark}"))
12247}
12248
12249fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12253 const MIN: usize = 20;
12254 for i in MIN..max.min(chars.len()) {
12255 if !matches!(chars[i], '.' | '!' | '?') {
12256 continue;
12257 }
12258 let Some(&next) = chars.get(i + 1) else {
12259 continue;
12260 };
12261 if !next.is_whitespace() {
12262 continue;
12263 }
12264 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12265 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12266 continue;
12267 }
12268 let word: String = chars[..i]
12269 .iter()
12270 .rev()
12271 .take_while(|c| !c.is_whitespace())
12272 .collect::<Vec<_>>()
12273 .into_iter()
12274 .rev()
12275 .collect();
12276 let word = word.to_ascii_lowercase();
12277 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12278 continue;
12279 }
12280 let end = chars[..i]
12281 .iter()
12282 .rposition(|c| !c.is_whitespace())
12283 .map_or(i, |p| p + 1);
12284 return Some(end);
12285 }
12286 None
12287}
12288
12289struct PrMessage {
12294 title: String,
12295 body: String,
12296}
12297
12298impl PrMessage {
12299 fn commit_message(&self) -> String {
12303 format!("{}\n\n{}", self.title, self.body)
12304 }
12305}
12306
12307fn title_marker(line: &str) -> Option<&str> {
12309 let line = line.trim();
12310 let head = line.get(..6)?;
12311 head.eq_ignore_ascii_case("title:")
12312 .then(|| line[6..].trim())
12313}
12314
12315fn summary_title(summary: &str) -> Option<String> {
12320 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12321 let raw = title_marker(first)?;
12322 if raw.is_empty() {
12323 return None;
12324 }
12325 let title = queue::title_from(raw, PR_TITLE_MAX);
12326 let lower = title.to_ascii_lowercase();
12327 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12328 return None;
12329 }
12330 Some(title)
12331}
12332
12333const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12335
12336pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12338
12339pub(crate) fn is_review_run(state: &RunState) -> bool {
12343 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12344}
12345
12346fn review_title(state: &RunState) -> Option<String> {
12351 english_subject(
12352 state.reviewed_commits.as_ref()?.first()?,
12353 state.config.graph.github_english(),
12354 )
12355}
12356
12357fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12360 let raw = raw.trim();
12361 if raw.is_empty() {
12362 return None;
12363 }
12364 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12365 return None;
12366 }
12367 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12368 return None;
12369 }
12370 let title = queue::title_from(raw, PR_TITLE_MAX);
12371 let lower = title.to_ascii_lowercase();
12372 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12373 return None;
12374 }
12375 Some(title)
12376}
12377
12378#[derive(Debug, Clone, PartialEq, Eq)]
12381struct BranchFacts {
12382 commits: Vec<(String, String)>,
12384 stat: String,
12386}
12387
12388const STAT_FILE_LINES: usize = 25;
12390const COMMITS_MAX_CHARS: usize = 20_000;
12392
12393async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12396 let commits = git::commit_log(repo, base, branch).await.ok()?;
12397 if commits.is_empty() {
12398 return None;
12399 }
12400 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12401 let lines: Vec<&str> = stat.lines().collect();
12402 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12403 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12404 let more = format!(" ... {omitted} more file(s)");
12405 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12406 kept.push(&more);
12407 kept.push(lines[lines.len() - 1]);
12408 kept.join("\n")
12409 } else {
12410 lines.join("\n")
12411 };
12412 Some(BranchFacts { commits, stat })
12413}
12414
12415fn markdown_safe(text: &str) -> String {
12418 text.replace("</details>", "</details>")
12419 .replace("\x60\x60\x60", "~~~")
12420}
12421
12422fn neutral_title(state: &RunState, winner: char) -> String {
12423 format!(
12424 "chore: land candidate {} of run {}",
12425 winner.to_ascii_uppercase(),
12426 state.id
12427 )
12428}
12429
12430pub(crate) async fn repair_landing_gate(
12433 state: &mut RunState,
12434 winner: &Candidate,
12435 outcomes: &[CommandOutcome],
12436) -> Result<GateFix> {
12437 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12438 return Ok(GateFix::Stop);
12439 }
12440 let mut runner = Runner {
12441 roles: state.config.resolve_roles()?,
12442 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12443 state: state.clone(),
12444 pause: Pause::new(),
12445 interrupt: Pause::new(),
12446 };
12447 let result = runner.gate_fix_round(winner, outcomes).await;
12448 *state = runner.state;
12449 result
12450}
12451
12452pub(crate) async fn verify_current_head(
12455 state: &mut RunState,
12456 winner: &Candidate,
12457 wait: &crate::repo_lock::WaitPolicy,
12458) -> Result<bool> {
12459 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12460 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12462 state
12463 .verification_history
12464 .push(crate::run::VerificationEvidence {
12465 superseded_by: Some(head.clone()),
12466 archived_at: Some(Timestamp::now()),
12467 fixes_spent: Some(state.gate_fixes.len()),
12468 gate_head: None,
12469 gate_ran: false,
12470 gate: Vec::new(),
12471 e2e: state.head_e2e.take(),
12472 });
12473 }
12474 if let Some(e) = &state.head_e2e
12475 && e.head == head
12476 {
12477 if e.passed {
12478 return Ok(true);
12479 }
12480 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12481 state.status = RunStatus::Blocked;
12482 state.save()?;
12483 return Ok(false);
12484 }
12485 } else if state.reviews.last().is_some_and(|r| {
12486 r.verified_head.as_deref() == Some(head.as_str())
12487 && (r.e2e_status() == E2eStatus::Passed
12488 || (r.e2e_status() == E2eStatus::NotConfigured
12489 && state.config.verify.e2e.is_empty()))
12490 }) || state
12491 .gate_repair_e2e
12492 .as_ref()
12493 .is_some_and(|e| e.head == head && e.passed)
12494 {
12495 return Ok(true);
12496 }
12497 if let Some(e) = &state.gate_repair_e2e
12498 && e.head == head
12499 && !e.passed
12500 && !e.outcomes.is_empty()
12501 && !verify_inconclusive(&e.outcomes)
12502 {
12503 state.status = RunStatus::Blocked;
12504 state.save()?;
12505 return Ok(false);
12506 }
12507 state.head_e2e = Some(crate::run::GateRepairE2e {
12508 head: head.clone(),
12509 passed: false,
12510 outcomes: Vec::new(),
12511 });
12512 state.save()?;
12513 let commands = state.config.verify.e2e.clone();
12514 let shell = state.config.shell();
12515 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12516 let cache_dir = state.config.cache_dir();
12517 let outcomes = if commands.is_empty() {
12518 Vec::new()
12519 } else {
12520 let (outcomes, _) = with_cache_lease(
12521 state,
12522 cache_dir.as_deref(),
12523 "e2e",
12524 "e2e",
12525 &winner.worktree,
12526 &head,
12527 timeout,
12528 "current head e2e",
12529 wait,
12530 |state, budget| async move {
12531 run_e2e_with_retry(
12532 state,
12533 &shell,
12534 &commands,
12535 &winner.worktree,
12536 budget,
12537 "current head e2e",
12538 )
12539 .await
12540 },
12541 )
12542 .await?;
12543 outcomes
12544 };
12545 let unresolved = verify_inconclusive(&outcomes);
12546 let passed = !unresolved
12547 && outcomes.iter().all(CommandOutcome::ok)
12548 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12549 if !outcomes.is_empty() && !unresolved {
12550 state.record_base_progress(&head);
12551 }
12552 state.head_e2e = Some(crate::run::GateRepairE2e {
12553 head: head.clone(),
12554 passed,
12555 outcomes,
12556 });
12557 if !passed && !unresolved {
12558 state.status = RunStatus::Blocked;
12559 state.event(
12560 "e2e",
12561 format!("verify.e2e failed on head {head}; nothing published"),
12562 );
12563 }
12564 state.save()?;
12565 Ok(passed)
12566}
12567
12568pub(crate) async fn check_gate_repair_e2e(
12571 state: &mut RunState,
12572 winner: &Candidate,
12573 wait: &crate::repo_lock::WaitPolicy,
12574) -> Result<GateFix> {
12575 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12576 return Ok(GateFix::Retry);
12577 }
12578 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12579 if state
12580 .gate_repair_e2e
12581 .as_ref()
12582 .is_some_and(|r| r.head == head && r.passed)
12583 {
12584 return Ok(GateFix::Retry);
12585 }
12586 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12587 head: head.clone(),
12588 passed: false,
12589 outcomes: Vec::new(),
12590 });
12591 state.save()?;
12592 let commands = state.config.verify.e2e.clone();
12593 let outcomes = if commands.is_empty() {
12594 Vec::new()
12595 } else {
12596 let shell = state.config.shell();
12597 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12598 let cache_dir = state.config.cache_dir();
12599 let (outcomes, _) = with_cache_lease(
12600 state,
12601 cache_dir.as_deref(),
12602 "e2e",
12603 "e2e",
12604 &winner.worktree,
12605 &head,
12606 timeout,
12607 "gate repair e2e",
12608 wait,
12609 |state, budget| async move {
12610 run_e2e_with_retry(
12611 state,
12612 &shell,
12613 &commands,
12614 &winner.worktree,
12615 budget,
12616 "gate repair e2e",
12617 )
12618 .await
12619 },
12620 )
12621 .await?;
12622 outcomes
12623 };
12624 let deferred = verify_inconclusive(&outcomes);
12625 let passed = outcomes.iter().all(CommandOutcome::ok)
12626 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12627 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12628 head,
12629 passed,
12630 outcomes,
12631 });
12632 state.save()?;
12633 if deferred {
12634 Ok(GateFix::Defer)
12635 } else if !passed {
12636 state.event(
12637 "gate",
12638 "verify.e2e failed after gate repair; nothing published",
12639 );
12640 Ok(GateFix::Stop)
12641 } else {
12642 Ok(GateFix::Retry)
12643 }
12644}
12645
12646pub(crate) async fn run_landing_gate(
12648 state: &mut RunState,
12649 winner: &Candidate,
12650 wait: &crate::repo_lock::WaitPolicy,
12651) -> Result<Vec<CommandOutcome>> {
12652 let shell = state.config.shell();
12653 let gate_commands = state.config.verify.gate.clone();
12654 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12655 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12656 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12657 Vec::new()
12658 } else {
12659 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12660 let cache_dir = state.config.cache_dir();
12661 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12662 let (outcomes, _) = with_cache_lease(
12663 state,
12664 cache_dir.as_deref(),
12665 "gate",
12666 "gate",
12667 &winner.worktree,
12668 &head,
12669 timeout,
12670 "final gate",
12671 wait,
12672 |state, budget| {
12673 let shell = shell.clone();
12674 let gate_commands = gate_commands.clone();
12675 let winner = winner.clone();
12676 async move {
12677 Runner::run_pre_gate(state, &winner).await;
12678 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12680 let (outcomes, timed_out_pids) = run_commands(
12681 state,
12682 "gate",
12683 "gate",
12684 0,
12685 &shell,
12686 &gate_commands,
12687 &winner.worktree,
12688 budget,
12689 )
12690 .await;
12691 (outcomes, false, timed_out_pids)
12692 }
12693 },
12694 )
12695 .await?;
12696 outcomes
12697 };
12698 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12699 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12700 let mut outcomes = outcomes;
12701 outcomes.push(CommandOutcome {
12702 command: "gate head stability".into(), code: None,
12703 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12704 duration_ms: 0, resource_blocked: false,
12705 });
12706 return Ok(outcomes);
12707 }
12708 if outcomes.is_empty() {
12709 state.event("gate", "no gate commands configured; not tested");
12714 }
12715 for o in &outcomes {
12716 state.event(
12717 "gate",
12718 format!(
12719 "`{}` -> {}",
12720 o.command,
12721 if o.ok() {
12722 "pass".to_owned()
12723 } else {
12724 format!(
12725 "FAIL ({:?})\n{}",
12726 o.code,
12727 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12728 )
12729 }
12730 ),
12731 );
12732 }
12733 Ok(outcomes)
12734}
12735
12736pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12742 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12743 ""
12744 } else {
12745 pr_title
12746 }
12747}
12748
12749pub fn landing_subject_source(state: &RunState) -> String {
12753 if is_review_run(state) {
12754 let winner = state.candidates.first().map_or('A', |c| c.label);
12755 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12756 }
12757 state.instruction.clone()
12758}
12759
12760fn summary_without_title(summary: &str) -> String {
12763 let mut lines = summary.trim().lines().peekable();
12764 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12765 lines.next();
12766 }
12767 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12768}
12769
12770#[cfg(test)]
12782fn pr_message(state: &RunState, winner: char) -> PrMessage {
12783 pr_message_with(state, winner, None)
12784}
12785
12786#[cfg(test)]
12789fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12790 let raw = pr_message_raw(
12791 state,
12792 winner,
12793 facts,
12794 crate::pr_outcome::FinalHead::Unreported,
12795 ChangeNote::NotAsked,
12796 );
12797 let id = crate::scrub::Identity::current();
12798 PrMessage {
12799 title: crate::scrub::scrub(&raw.title, &id),
12800 body: crate::scrub::scrub(&raw.body, &id),
12801 }
12802}
12803
12804#[derive(Debug, Clone, Copy)]
12806enum ChangeNote<'a> {
12807 NotAsked,
12809 Summary(&'a crate::run::ChangeSummary),
12811 Unavailable(&'a str),
12813}
12814
12815fn pr_message_raw(
12816 state: &RunState,
12817 winner: char,
12818 facts: Option<&BranchFacts>,
12819 head: crate::pr_outcome::FinalHead<'_>,
12820 change: ChangeNote<'_>,
12821) -> PrMessage {
12822 if !matches!(change, ChangeNote::NotAsked) {
12823 let (title, mut body) = match change {
12824 ChangeNote::Summary(c) => (
12825 c.title.clone(),
12826 format!("## Summary\n\n{}\n\n", c.body.trim()),
12827 ),
12828 ChangeNote::Unavailable(why) => (
12829 neutral_title(state, winner),
12830 format!(
12831 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12832 ),
12833 ),
12834 ChangeNote::NotAsked => unreachable!(),
12835 };
12836 body.push_str(&public_pr_evidence(state, head));
12837 body.push_str(&format!(
12838 "\n---\nmagi:run/{} magi:candidate-{}\n",
12839 state.id,
12840 winner.to_ascii_lowercase()
12841 ));
12842 return PrMessage { title, body };
12843 }
12844 pr_preview(state, winner, facts, head)
12845}
12846
12847fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12850 let mut body = crate::pr_outcome::review_rounds(state);
12851 body.push_str(&crate::pr_outcome::final_verification(state, head));
12852 let open = state.last_round_findings();
12853 if !open.is_empty() {
12854 body.push_str("## Open review findings\n\n");
12855 for f in open {
12856 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12857 }
12858 body.push('\n');
12859 }
12860 if let Some(c) = state.change_summary.as_ref()
12861 && head == crate::pr_outcome::FinalHead::At(&c.head)
12862 {
12863 body.push_str(&format!(
12864 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12865 c.base, c.head
12866 ));
12867 }
12868 body
12869}
12870
12871fn pr_preview(
12873 state: &RunState,
12874 winner: char,
12875 facts: Option<&BranchFacts>,
12876 head: crate::pr_outcome::FinalHead<'_>,
12877) -> PrMessage {
12878 let summary = state
12879 .candidates
12880 .iter()
12881 .find(|c| c.label == winner)
12882 .map(|c| c.summary.as_str())
12883 .unwrap_or_default();
12884 let review = is_review_run(state);
12887 let title = if review {
12888 facts
12889 .and_then(|f| {
12890 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12891 })
12892 .or_else(|| review_title(state))
12893 .or_else(|| {
12894 state
12895 .candidates
12896 .iter()
12897 .find(|c| c.label == winner)
12898 .filter(|c| !c.branch.starts_with("magi/"))
12899 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12900 })
12901 .unwrap_or_else(|| neutral_title(state, winner))
12902 } else {
12903 summary_title(summary).unwrap_or_else(|| {
12904 english_title(
12905 &state.instruction,
12906 PR_TITLE_MAX,
12907 state.config.graph.github_english(),
12908 )
12909 .unwrap_or_else(|| neutral_title(state, winner))
12910 })
12911 };
12912
12913 let mut body = String::new();
12914 let what = summary_without_title(summary);
12915 if !what.is_empty() {
12916 body.push_str("## Summary\n\n");
12917 body.push_str(&what);
12918 body.push_str("\n\n");
12919 }
12920
12921 body.push_str(&crate::pr_outcome::review_rounds(state));
12923 body.push_str(&crate::pr_outcome::final_verification(state, head));
12924 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12925
12926 let open = state.open_findings();
12927 if !open.is_empty() {
12928 body.push_str("## Open review findings\n\n");
12929 for f in &open {
12930 body.push_str(&format!(
12931 "- `{}` [{:?}] {}\n",
12932 f.id, f.severity, "Context retained in local review records."
12933 ));
12934 }
12935 body.push('\n');
12936 }
12937
12938 if let Some(fix) = fix
12939 && !fix.rejected.is_empty()
12940 {
12941 body.push_str("## Declined by the fixer\n\n");
12942 for r in &fix.rejected {
12943 body.push_str(&format!(
12944 "- `{}`: {}\n",
12945 r.id, "Reason retained in local review records."
12946 ));
12947 }
12948 body.push('\n');
12949 }
12950
12951 if review {
12952 body.push_str("## Commits under review\n\n");
12954 if let Some(facts) = facts {
12955 let mut left = COMMITS_MAX_CHARS;
12956 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12957 let mut entry = format!("- {}\n", markdown_safe(subject));
12958 for l in markdown_safe(text).lines() {
12959 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12960 }
12961 if left == 0 {
12962 body.push_str(&format!(
12963 "- ... {} more commit(s)\n",
12964 facts.commits.len() - i
12965 ));
12966 break;
12967 }
12968 if entry.len() > left {
12969 let mut end = left;
12972 while !entry.is_char_boundary(end) {
12973 end -= 1;
12974 }
12975 entry.truncate(end);
12976 entry.push_str("\n ... (truncated)\n");
12977 left = 0;
12978 } else {
12979 left -= entry.len();
12980 }
12981 body.push_str(&entry);
12982 }
12983 if !facts.stat.trim().is_empty() {
12984 body.push_str(&format!(
12985 "\n## Diff stat\n\n```\n{}\n```\n",
12986 markdown_safe(facts.stat.trim())
12987 ));
12988 }
12989 } else {
12990 match &state.reviewed_commits {
12991 Some(subjects) => {
12992 for s in subjects {
12993 body.push_str(&format!("- {}\n", s.trim()));
12994 }
12995 }
12996 None => {
12997 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13000 body.push_str(rest.trim());
13001 body.push('\n');
13002 }
13003 }
13004 }
13005 } else {
13006 let task = state.instruction.trim();
13007 let task = if task.is_empty() {
13008 "(empty task)"
13009 } else {
13010 task
13011 };
13012 body.push_str(&format!(
13013 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13014 task.replace("</details>", "</details>")
13015 ));
13016 }
13017
13018 body.push_str(&format!(
13019 "\n---\nmagi:run/{} magi:candidate-{}\n",
13020 state.id,
13021 winner.to_ascii_lowercase()
13022 ));
13023
13024 PrMessage { title, body }
13025}
13026
13027fn seeded_instruction(state: &RunState) -> String {
13030 let mut inputs = Vec::new();
13031 if let Some(start) = &state.sources.start_from {
13032 inputs.push(format!(
13033 "Start from `{}` at `{}` (including ancestry).",
13034 start.token, start.sha
13035 ));
13036 }
13037 for pick in &state.sources.pick_commits {
13038 inputs.push(format!(
13039 "Pick `{}` at `{}` (single commit).",
13040 pick.token, pick.sha
13041 ));
13042 }
13043 if inputs.is_empty() {
13044 state.instruction.clone()
13045 } else {
13046 format!(
13047 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13048 state.instruction,
13049 inputs.join("\n")
13050 )
13051 }
13052}
13053
13054async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13058 let base = &state.base_branch;
13059 let mut against = base.clone();
13060 if mode == MergeMode::Pr {
13061 let remote = &state.config.merge.remote;
13065 let tracking = format!("{remote}/{base}");
13066 let fetched = git::fetch(repo, remote, base).await;
13067 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13068 against = tracking;
13069 } else {
13070 return false;
13071 }
13072 }
13073 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13074}
13075
13076fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13079 let mut detail = format!(
13080 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13081 nothing to open a pull request for"
13082 );
13083 detail.push_str(
13084 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13085 );
13086 detail
13087}
13088
13089#[derive(Debug, PartialEq, Eq)]
13092enum PrPlan {
13093 Create,
13094 Adopt { url: String, title: String },
13095 Stop(String),
13096}
13097
13098fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13101 match found {
13102 Ok(land::OpenPr::None) => PrPlan::Create,
13103 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13104 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13105 "several open pull requests exist for this branch, not picking one: {}",
13106 urls.join(" ")
13107 )),
13108 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13109 }
13110}
13111
13112pub(crate) async fn gh_pr_create(
13114 cwd: &Path,
13115 base: &str,
13116 head: &str,
13117 title: &str,
13118 body: &str,
13119) -> Result<String> {
13120 let out = tokio::process::Command::new("gh")
13121 .args([
13122 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13123 ])
13124 .current_dir(cwd)
13125 .quiet()
13126 .stdin(std::process::Stdio::null())
13127 .output()
13128 .await
13129 .context("spawn gh")?;
13130 if out.status.success() {
13131 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13132 } else {
13133 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13134 }
13135}
13136
13137async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13139 match std::fs::symlink_metadata(path) {
13140 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13141 git::worktree_remove_clean(repo, path).await?;
13144 return Ok(());
13145 }
13146 Err(e) => return Err(e.into()),
13147 Ok(_) => {}
13148 }
13149 if !git::is_clean(path).await? {
13150 bail!(
13151 "worktree has uncommitted work; refusing removal: {}",
13152 path.display()
13153 );
13154 }
13155 if !git::worktree_remove_clean(repo, path).await? {
13156 bail!("worktree removal refused: {}", path.display());
13157 }
13158 Ok(())
13159}
13160
13161pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13170 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13171 bail!("refusing fold: {reason}");
13172 }
13173 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13174 bail!("candidate preservation incomplete; refusing cleanup");
13175 }
13176 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13180 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13181 let _driver = crate::driver::maintenance(home, &state.id)?;
13182 let mut output_guards = Vec::new();
13183 for entry in std::fs::read_dir(state.worktree_root())
13184 .into_iter()
13185 .flatten()
13186 .flatten()
13187 {
13188 if entry.path().join(".git").is_file() {
13189 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13190 }
13191 }
13192 let repo = state.repo.clone();
13193 let wait = repository_wait(state, None);
13196 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13197 Ok(lease) => lease,
13198 Err(e) => {
13199 state.save_under(home)?;
13200 return Err(e);
13201 }
13202 };
13203 let _source_outputs = if state.status == RunStatus::Merged
13206 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13207 {
13208 verify_merged_fold_sources(state, drop_winner, home).await?
13209 } else {
13210 Vec::new()
13211 };
13212 let root = state.worktree_root();
13213 let winner = state.tally.as_ref().map(|t| t.winner);
13214 let mut removed = Vec::new();
13215
13216 for i in 0..state.candidates.len() {
13217 let c = state.candidates[i].clone();
13218 let is_winner = Some(c.label) == winner;
13219 if is_winner && !drop_winner {
13220 continue;
13221 }
13222 let existed = c.worktree.exists();
13223 remove_preserved_worktree(&repo, &c.worktree).await?;
13226 if existed {
13227 removed.push(c.worktree.to_string_lossy().into_owned());
13228 }
13229 let handed_over = state.released_branches.contains(&c.branch);
13232 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13233 if !git::branch_delete(&repo, &c.branch).await? {
13234 bail!("candidate branch removal refused");
13235 }
13236 removed.push(c.branch.clone());
13237 }
13238 state.candidates[i].folded = true;
13239 }
13240
13241 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13242 let path = name.path();
13243 let keep = !drop_winner
13244 && winner.is_some_and(|w| {
13245 path.file_name()
13246 .is_some_and(|n| n == format!("cand-{w}").as_str())
13247 });
13248 if keep {
13249 continue;
13250 }
13251 remove_preserved_worktree(&repo, &path).await?;
13252 removed.push(path.to_string_lossy().into_owned());
13253 }
13254
13255 remove_if_empty(&root);
13264
13265 if state.enabled_worktree_config && drop_winner {
13266 git::release_worktree_config(&repo).await.ok();
13270 state.enabled_worktree_config = false;
13271 }
13272 state.save_under(home)?;
13273 Ok(removed)
13274}
13275
13276async fn verify_merged_fold_sources(
13277 state: &RunState,
13278 drop_winner: bool,
13279 home: &Path,
13280) -> Result<Vec<std::fs::File>> {
13281 let winner = state.tally.as_ref().map(|t| t.winner);
13282 let mut heads = BTreeSet::new();
13283 for candidate in &state.candidates {
13284 if Some(candidate.label) == winner && !drop_winner {
13285 continue;
13286 }
13287 anyhow::ensure!(
13288 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13289 "candidate recovery source is outside the inventoried run root: {}",
13290 candidate.worktree.display()
13291 );
13292 match std::fs::symlink_metadata(&candidate.worktree) {
13293 Ok(_) => {}
13294 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13295 Err(error) => return Err(error.into()),
13296 }
13297 if !state.released_branches.contains(&candidate.branch)
13298 && git::branch_exists(&state.repo, &candidate.branch).await?
13299 {
13300 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13301 }
13302 }
13303 let mut cargo_guards = Vec::new();
13304 let entries = match std::fs::read_dir(state.worktree_root()) {
13305 Ok(entries) => Some(entries),
13306 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13307 Err(error) => return Err(error.into()),
13308 };
13309 for entry in entries.into_iter().flatten() {
13310 let path = entry?.path();
13311 let metadata = std::fs::symlink_metadata(&path)?;
13312 anyhow::ensure!(
13313 metadata.is_dir() && !metadata.file_type().is_symlink(),
13314 "non-directory or symlink recovery source retained: {}",
13315 path.display()
13316 );
13317 if !drop_winner
13318 && winner.is_some_and(|w| {
13319 path.file_name()
13320 .is_some_and(|name| name == format!("cand-{w}").as_str())
13321 })
13322 {
13323 continue;
13324 }
13325 anyhow::ensure!(
13326 path.join(".git").is_file(),
13327 "unregistered recovery source retained: {}",
13328 path.display()
13329 );
13330 anyhow::ensure!(
13331 git::git(
13332 &path,
13333 &[
13334 "status",
13335 "--porcelain",
13336 "--untracked-files=all",
13337 "--ignored",
13338 "--",
13339 ".",
13340 ":(exclude)target"
13341 ]
13342 )
13343 .await?
13344 .is_empty(),
13345 "source changes or ignored recovery files retained: {}",
13346 path.display()
13347 );
13348 match std::fs::symlink_metadata(path.join("target")) {
13349 Ok(_) => {
13350 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13351 }
13352 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13353 Err(error) => return Err(error.into()),
13354 }
13355 for checkpoint in [
13356 "rebase-merge",
13357 "rebase-apply",
13358 "sequencer",
13359 "CHERRY_PICK_HEAD",
13360 "MERGE_HEAD",
13361 "REBASE_HEAD",
13362 "REVERT_HEAD",
13363 "BISECT_LOG",
13364 "AUTO_MERGE",
13365 "MERGE_MSG",
13366 ] {
13367 let location = git::git(
13368 &path,
13369 &[
13370 "rev-parse",
13371 "--path-format=absolute",
13372 "--git-path",
13373 checkpoint,
13374 ],
13375 )
13376 .await?;
13377 anyhow::ensure!(
13378 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13379 if e.kind() == std::io::ErrorKind::NotFound),
13380 "Git recovery checkpoint retained: {checkpoint}"
13381 );
13382 }
13383 heads.insert(git::rev_parse(&path, "HEAD").await?);
13384 }
13385 crate::fold_evidence::preserve(state, home, &heads).await?;
13386
13387 Ok(cargo_guards)
13388}
13389
13390fn remove_if_empty(dir: &Path) {
13401 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13402 std::fs::remove_dir(dir).ok();
13403 }
13404}
13405
13406pub fn worst_open(state: &RunState) -> Option<Severity> {
13408 state
13409 .reviews
13410 .last()?
13411 .reviews
13412 .iter()
13413 .flat_map(|r| r.findings.iter())
13414 .map(|f| f.severity)
13415 .max()
13416}
13417
13418async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13422 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13423 git::acquire_worktree_config(repo).await?;
13424 state.enabled_worktree_config = true;
13425 }
13426 Ok(())
13427}
13428
13429#[cfg(test)]
13430mod tests {
13431 #[tokio::test]
13432 async fn launcher_base_fetch_requires_remote_success() {
13433 let tmp = tempfile::tempdir().unwrap();
13434 let repo = tmp.path().join("repo");
13435 let remote = tmp.path().join("remote");
13436 std::fs::create_dir(&repo).unwrap();
13437 std::fs::create_dir(&remote).unwrap();
13438 git::git(&remote, &["init", "--bare", "-b", "main"])
13439 .await
13440 .unwrap();
13441 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13442 git::git(
13443 &repo,
13444 &[
13445 "-c",
13446 "user.name=Fixture",
13447 "-c",
13448 "user.email=fixture@example.invalid",
13449 "commit",
13450 "--allow-empty",
13451 "-m",
13452 "fixture",
13453 ],
13454 )
13455 .await
13456 .unwrap();
13457 git::git(
13458 &repo,
13459 &["remote", "add", "origin", remote.to_str().unwrap()],
13460 )
13461 .await
13462 .unwrap();
13463 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13464 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13465 assert_eq!(
13466 resolve_base(&repo, "main", "origin", None)
13467 .await
13468 .unwrap()
13469 .commit,
13470 head
13471 );
13472 #[cfg(unix)]
13473 {
13474 let policy: crate::config::Coordination = toml::from_str(
13475 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13476 ).unwrap();
13477 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13478 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13479 let held = crate::repo_lock::SeatLease::operation_with(
13480 &repo,
13481 Some(&policy),
13482 &crate::repo_lock::WaitPolicy::default(),
13483 )
13484 .await
13485 .unwrap();
13486 assert!(held.is_some());
13487 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13488 tokio::pin!(fetch);
13489 assert!(
13490 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13491 .await
13492 .is_err()
13493 );
13494 assert!(
13495 !repo.join(".git/FETCH_HEAD").exists(),
13496 "no fetch before admission"
13497 );
13498 drop(held);
13499 let fetched = fetch.await.unwrap();
13500 assert_eq!(fetched.commit, head);
13501 assert_eq!(fetched.remote, "origin");
13502 assert_eq!(fetched.branch, "main");
13503 }
13504 git::git(
13506 &repo,
13507 &[
13508 "remote",
13509 "set-url",
13510 "origin",
13511 tmp.path().join("missing").to_str().unwrap(),
13512 ],
13513 )
13514 .await
13515 .unwrap();
13516 let error = resolve_base(&repo, "main", "origin", None)
13517 .await
13518 .unwrap_err()
13519 .to_string();
13520 assert!(error.contains("cannot read origin/main"));
13521 assert!(error.contains("refusing to branch off"));
13522 }
13523
13524 #[test]
13525 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13526 let leaked = vec!["chore(deps): update a crate".to_owned()];
13527 let own = "fix(daemon): apply a chosen action";
13528 assert!(should_retitle("", own, &leaked));
13529 assert!(should_retitle(
13530 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13531 own,
13532 &leaked
13533 ));
13534 assert!(should_retitle(
13535 "chore: land candidate A of run 1",
13536 own,
13537 &leaked
13538 ));
13539 assert!(should_retitle(
13540 "magi: candidate A (uncommitted work)",
13541 own,
13542 &leaked
13543 ));
13544 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13545 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13546 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13547 }
13548
13549 #[test]
13550 fn pr_merge_plan_creates_adopts_or_stops() {
13551 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13552 assert_eq!(
13553 pr_merge_plan(Ok(land::OpenPr::One {
13554 url: "u".into(),
13555 title: "t".into()
13556 })),
13557 PrPlan::Adopt {
13558 url: "u".into(),
13559 title: "t".into()
13560 }
13561 );
13562 let PrPlan::Stop(many) =
13563 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13564 else {
13565 panic!("many must stop");
13566 };
13567 assert!(many.contains('a') && many.contains('b'));
13568 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13569 panic!("a failed lookup must stop");
13570 };
13571 assert!(err.contains("bad token"));
13572 }
13573
13574 use super::*;
13575 use crate::run::GateStatus;
13576 use std::collections::BTreeMap;
13577 use std::time::Duration;
13578
13579 fn conductor() -> AgentSpec {
13580 AgentSpec {
13581 id: "conductor".to_owned(),
13582 kind: crate::config::AgentKind::Command,
13583 model: None,
13584 command: vec!["true".to_owned()],
13585 extra_args: Vec::new(),
13586 env: BTreeMap::new(),
13587 prompt_delivery: None,
13588 max_turn_secs: None,
13589 }
13590 }
13591
13592 fn spec(id: &str) -> AgentSpec {
13593 AgentSpec {
13594 id: id.to_owned(),
13595 kind: crate::config::AgentKind::Command,
13596 model: None,
13597 command: vec!["true".to_owned()],
13598 extra_args: Vec::new(),
13599 env: BTreeMap::new(),
13600 prompt_delivery: None,
13601 max_turn_secs: None,
13602 }
13603 }
13604
13605 fn ids(xs: &[&str]) -> BTreeSet<String> {
13606 xs.iter().map(|s| (*s).to_owned()).collect()
13607 }
13608
13609 #[test]
13610 fn pick_successor_skips_an_agent_another_seat_holds() {
13611 let roster = [spec("a"), spec("b"), spec("c")];
13613 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13614 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13615 }
13616
13617 #[test]
13618 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13619 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13621 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13622 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13623 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13624 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13625 }
13626
13627 #[test]
13628 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13629 let roster = [spec("a"), spec("b")];
13630 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13631 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13632 let carried = ids(&["b"]);
13633 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13634 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13635 }
13636
13637 #[test]
13638 fn pick_successor_returns_none_without_any_untried_successor() {
13639 let roster = [spec("a"), spec("b")];
13640 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13641 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13642 let carried = ids(&["a"]);
13643 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13644 }
13645
13646 #[test]
13647 fn pick_successor_rescue_avoids_another_seats_occupant() {
13648 let roster = [spec("a"), spec("b"), spec("c")];
13650 let carried = ids(&["b", "c"]);
13651 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13652 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13653 }
13654
13655 #[test]
13656 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13657 let roster = [spec("a"), spec("b"), spec("c")];
13658 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13659 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13660 }
13661
13662 #[test]
13663 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13664 let roster = [spec("a"), spec("b"), spec("c")];
13665 let failed = ids(&["a", "b", "c"]);
13666 let mut tried = ids(&["b"]);
13668 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13669 assert_eq!(first.id, "a");
13670 tried.insert(first.id.clone());
13671 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13672 assert_eq!(second.id, "c");
13673 tried.insert(second.id.clone());
13674 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13675 }
13676
13677 #[test]
13678 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13679 let roster = [spec("a"), spec("b")];
13680 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13681 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13682 }
13683
13684 #[test]
13685 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13686 let roster = [spec("a"), spec("b"), spec("c")];
13687 let h = SeatHistory {
13688 failed: ids(&["a"]),
13689 last_ok: Some("b".to_owned()),
13690 last_fail: None,
13691 };
13692 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13693 let gone = SeatHistory {
13695 last_ok: Some("zzz".to_owned()),
13696 ..h.clone()
13697 };
13698 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13699 let failed = SeatHistory {
13700 failed: ids(&["a", "b"]),
13701 last_ok: Some("b".to_owned()),
13702 last_fail: None,
13703 };
13704 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13705 }
13706
13707 #[test]
13708 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13709 let first = SeatState::new("review-1", "alpha", 7);
13710 let next = handover_seat("review-1", "gamma", 7);
13711 assert_ne!(first.claude_session, next.claude_session);
13712 }
13713
13714 #[test]
13715 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13716 let mut state = state_with_summary("x", "y");
13717 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13718 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13719 assert_ne!(a.claude_session, b.claude_session);
13720 }
13721
13722 #[test]
13723 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13724 let roster = [spec("a"), spec("b")];
13725 let h = SeatHistory {
13726 failed: ids(&["a", "b"]),
13727 ..SeatHistory::default()
13728 };
13729 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13730 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13731 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13732 }
13733
13734 #[test]
13741 fn failure_signature_ignores_numbers_and_paths() {
13742 assert_eq!(
13743 failure_signature("exited with Some(2) and no usable output"),
13744 failure_signature("exited with Some(137) and no usable output")
13745 );
13746 assert_eq!(
13747 failure_signature("process exited with no usable output (exit code 2)"),
13748 failure_signature("process exited with no usable output (exit code 137)")
13749 );
13750 for (old, new) in [
13751 (
13752 "exited with Some(2) and no usable output",
13753 "process exited with no usable output (exit code 2)",
13754 ),
13755 (
13756 "exited with None and no usable output",
13757 "process exited with no usable output (exit code unavailable)",
13758 ),
13759 ] {
13760 let previous = FailClass::Other(failure_signature(old));
13761 let current = FailClass::Other(failure_signature(new));
13762 assert_eq!(previous, current, "resumption retains the failure class");
13763 assert!(!should_hand_over(Some(&previous), ¤t));
13764 }
13765 assert_eq!(
13766 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13767 failure_signature("cannot open /var/x.txt: denied")
13768 );
13769 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13770 }
13771
13772 #[test]
13773 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13774 let other = FailClass::Other("x".into());
13775 assert!(should_hand_over(None, &FailClass::Quota));
13776 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13777 assert!(should_hand_over(
13778 Some(&FailClass::Timeout),
13779 &FailClass::Timeout
13780 ));
13781 assert!(should_hand_over(None, &other));
13782 assert!(!should_hand_over(Some(&other), &other));
13783 assert!(should_hand_over(
13784 Some(&other),
13785 &FailClass::Other("y".into())
13786 ));
13787 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13789 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13790 }
13791
13792 #[test]
13793 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13794 let a = SeatState::new("judge-1", "alpha", 7);
13795 let b = handover_seat("judge-1", "beta", 7);
13796 assert_ne!(a.claude_session, b.claude_session);
13797 assert_eq!(b.turns, 0);
13798 }
13799
13800 #[test]
13801 fn a_timeout_is_classified_apart_from_other_failures() {
13802 assert_eq!(
13803 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13804 Some(FailClass::Timeout)
13805 );
13806 assert!(matches!(
13807 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13808 Some(FailClass::Other(_))
13809 ));
13810 }
13811
13812 #[test]
13813 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13814 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13815 let tried = BTreeSet::from(["beta".to_owned()]);
13816 let next = next_untried_in_roster(&roster, 1, &tried);
13819 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13820 }
13821
13822 #[test]
13823 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13824 let roster = vec![spec("alpha"), spec("beta")];
13825 let tried = BTreeSet::from(["beta".to_owned()]);
13826 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13830 }
13831
13832 #[test]
13833 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13834 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13835 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13836 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13840 }
13841
13842 #[test]
13843 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13844 let roster = vec![spec("a"), spec("a"), spec("b")];
13845 let tried = BTreeSet::from(["a".to_owned()]);
13846 let next = next_untried_in_roster(&roster, 0, &tried);
13847 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13848 }
13849
13850 #[test]
13851 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13852 let roster = vec![spec("a"), spec("b")];
13853 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13854 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13855 }
13856
13857 #[test]
13858 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13859 let dir = tempfile::tempdir().unwrap();
13860 let bay = dir.path().join("ffff");
13861
13862 remove_if_empty(&bay);
13864 assert!(!bay.exists());
13865
13866 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13869 remove_if_empty(&bay);
13870 assert!(bay.exists(), "non-empty directory must survive");
13871
13872 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13874 remove_if_empty(&bay);
13875 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13876 }
13877
13878 #[test]
13887 fn a_full_panel_that_found_nothing_is_clean() {
13888 assert!(round_is_clean(
13889 0,
13890 true,
13891 2,
13892 2,
13893 0,
13894 IncompleteReviewPolicy::Block
13895 ));
13896 }
13897
13898 #[test]
13899 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13900 assert!(!round_is_clean(
13901 0,
13902 true,
13903 1,
13904 2,
13905 0,
13906 IncompleteReviewPolicy::Block
13907 ));
13908 }
13909
13910 #[test]
13911 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13912 assert!(!round_is_clean(
13913 1,
13914 true,
13915 1,
13916 2,
13917 0,
13918 IncompleteReviewPolicy::Warn
13919 ));
13920 }
13921
13922 #[test]
13923 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13924 assert!(round_is_clean(
13925 0,
13926 true,
13927 1,
13928 2,
13929 0,
13930 IncompleteReviewPolicy::Warn
13931 ));
13932 }
13933
13934 #[test]
13935 fn a_full_panel_with_an_open_finding_is_not_clean() {
13936 assert!(!round_is_clean(
13937 1,
13938 true,
13939 2,
13940 2,
13941 0,
13942 IncompleteReviewPolicy::Block
13943 ));
13944 }
13945
13946 #[test]
13947 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13948 assert!(!round_is_clean(
13949 0,
13950 false,
13951 2,
13952 2,
13953 0,
13954 IncompleteReviewPolicy::Block
13955 ));
13956 }
13957
13958 #[test]
13965 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13966 assert!(round_is_clean(
13969 0,
13970 true,
13971 1,
13972 2,
13973 1,
13974 IncompleteReviewPolicy::Block
13975 ));
13976 }
13977
13978 #[test]
13979 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13980 assert!(!round_is_clean(
13983 0,
13984 true,
13985 1,
13986 2,
13987 0,
13988 IncompleteReviewPolicy::Block
13989 ));
13990 }
13991
13992 #[test]
13993 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13994 assert!(!round_is_clean(
13995 1,
13996 true,
13997 1,
13998 2,
13999 1,
14000 IncompleteReviewPolicy::Block
14001 ));
14002 assert!(!round_is_clean(
14003 0,
14004 false,
14005 1,
14006 2,
14007 1,
14008 IncompleteReviewPolicy::Block
14009 ));
14010 }
14011
14012 #[test]
14013 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14014 assert!(!round_is_clean(
14018 0,
14019 true,
14020 0,
14021 2,
14022 2,
14023 IncompleteReviewPolicy::Block
14024 ));
14025 }
14026
14027 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14028 CommandOutcome {
14029 command: "test".to_owned(),
14030 code,
14031 output_tail: String::new(),
14032 duration_ms: 0,
14033 resource_blocked,
14034 }
14035 }
14036
14037 #[cfg(unix)]
14038 #[tokio::test]
14039 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14040 ask_test_home();
14041 let temp = tempfile::tempdir().unwrap();
14042 let mut runner = runner_at(RunStatus::Gating);
14043 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14044 let commands = vec![
14045 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14046 "touch must-not-run".to_owned(),
14047 ];
14048 let (outcomes, retried, pids) = run_e2e_with_retry(
14049 &mut runner.state,
14050 &shell,
14051 &commands,
14052 temp.path(),
14053 Duration::from_secs(5),
14054 "fixture",
14055 )
14056 .await;
14057 assert_eq!(outcomes.len(), 1);
14058 assert!(outcomes[0].resource_blocked);
14059 assert!(!outcomes[0].build_failed());
14060 assert!(!retried);
14061 assert!(pids.is_empty());
14062 assert!(!temp.path().join("must-not-run").exists());
14063 assert_eq!(
14064 runner.state.resource_block.as_ref().unwrap().kind,
14065 "incremental cache admission"
14066 );
14067 }
14068
14069 #[test]
14070 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14071 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14072 assert!(
14073 !verify_inconclusive(&[outcome(Some(1), false)]),
14074 "an ordinary failure is still evidence about the patch"
14075 );
14076 assert!(verify_inconclusive(&[outcome(None, true)]));
14077 assert!(
14078 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14079 "one inconclusive outcome taints the whole batch"
14080 );
14081 assert!(!verify_inconclusive(&[]));
14082 }
14083
14084 #[tokio::test]
14085 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14086 let calls = std::sync::atomic::AtomicUsize::new(0);
14090 let started = Instant::now();
14091 wait_for_pids_with(
14092 &[123],
14093 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14094 Duration::from_millis(5),
14095 Duration::from_secs(5),
14096 )
14097 .await;
14098 assert!(
14099 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14100 "must keep checking rather than deciding on the first answer"
14101 );
14102 assert!(
14103 started.elapsed() < Duration::from_secs(1),
14104 "must return the moment it is confirmed dead, not wait out the ceiling"
14105 );
14106 }
14107
14108 #[tokio::test]
14109 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14110 let started = Instant::now();
14111 wait_for_pids_with(
14112 &[123],
14113 |_| true, Duration::from_millis(5),
14115 Duration::from_millis(30),
14116 )
14117 .await;
14118 let elapsed = started.elapsed();
14119 assert!(
14120 elapsed >= Duration::from_millis(30),
14121 "must not give up before its own ceiling: {elapsed:?}"
14122 );
14123 assert!(
14124 elapsed < Duration::from_secs(1),
14125 "must not wait past its own ceiling either: {elapsed:?}"
14126 );
14127 }
14128
14129 #[tokio::test]
14130 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14131 let started = Instant::now();
14132 wait_for_pids_with(
14133 &[],
14134 |_| true,
14135 Duration::from_secs(5),
14136 Duration::from_secs(5),
14137 )
14138 .await;
14139 assert!(
14140 started.elapsed() < Duration::from_millis(200),
14141 "an empty pid list has nothing to confirm"
14142 );
14143 }
14144
14145 fn review_round(
14151 clean: bool,
14152 blocking: usize,
14153 answered: usize,
14154 expected: usize,
14155 progressed: bool,
14156 e2e_ok: bool,
14157 ) -> ReviewRound {
14158 ReviewRound {
14159 round: 1,
14160 head: "h".to_owned(),
14161 verified_head: None,
14162 verified_at: None,
14163 reviews: Vec::new(),
14164 e2e: vec![CommandOutcome {
14165 command: "test".to_owned(),
14166 code: Some(if e2e_ok { 0 } else { 1 }),
14167 output_tail: String::new(),
14168 duration_ms: 0,
14169 resource_blocked: false,
14170 }],
14171 verify_retried: false,
14172 e2e_deferred: false,
14173 e2e_defer_reason: None,
14174 fix: None,
14175 blocking,
14176 answered,
14177 expected,
14178 clean,
14179 progressed,
14180 vote_split: false,
14181 reconsideration: Vec::new(),
14182 verdict: None,
14183 }
14184 }
14185
14186 #[test]
14187 fn review_conclusion_is_none_when_nothing_has_run() {
14188 assert_eq!(review_conclusion(&[], 3), None);
14189 }
14190
14191 #[test]
14192 fn review_conclusion_is_none_while_rounds_remain() {
14193 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14194 assert_eq!(review_conclusion(&rounds, 3), None);
14195 }
14196
14197 #[test]
14198 fn review_conclusion_is_gating_once_a_round_is_clean() {
14199 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14200 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14201 }
14202
14203 #[test]
14204 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14205 let rounds = vec![
14206 review_round(false, 1, 2, 2, true, true),
14207 review_round(false, 1, 2, 2, true, true),
14208 ];
14209 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14210 }
14211
14212 #[test]
14213 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14214 let rounds = vec![
14215 review_round(false, 1, 2, 2, true, true),
14216 review_round(false, 1, 2, 2, true, false),
14217 ];
14218 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14219 }
14220
14221 #[test]
14222 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14223 let mut blocked = review_round(false, 1, 2, 2, true, false);
14230 blocked.e2e[0].resource_blocked = true;
14231 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14232 assert_eq!(review_conclusion(&rounds, 2), None);
14233 }
14234
14235 #[test]
14236 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14237 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14239 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14240 }
14241
14242 #[test]
14243 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14244 let rounds = vec![
14245 review_round(false, 1, 2, 2, false, true),
14246 review_round(false, 1, 2, 2, false, true),
14247 ];
14248 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14249 }
14250
14251 fn secs(n: u64) -> Duration {
14252 Duration::from_secs(n)
14253 }
14254
14255 #[tokio::test]
14258 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14259 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14260 if std::env::var_os(CHILD).is_none() {
14261 let output = std::process::Command::new(std::env::current_exe().unwrap())
14265 .args([
14266 "--exact",
14267 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14268 "--nocapture",
14269 ])
14270 .env(CHILD, "1")
14271 .env_remove("MAGI_WORKER_HOME")
14272 .env_remove("MAGI_WORKER_ID")
14273 .output()
14274 .unwrap();
14275 assert!(
14276 output.status.success()
14277 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14278 "{}\n{}",
14279 String::from_utf8_lossy(&output.stdout),
14280 String::from_utf8_lossy(&output.stderr)
14281 );
14282 return;
14283 }
14284 ask_test_home();
14285 let tmp = tempfile::tempdir().unwrap();
14286 let repo = tmp.path().join("repo");
14287 std::fs::create_dir_all(&repo).unwrap();
14288 init_repo(&repo);
14289 let g = |args: &[&str]| {
14290 let out = std::process::Command::new("git")
14291 .args(args)
14292 .current_dir(&repo)
14293 .quiet()
14294 .output()
14295 .unwrap();
14296 assert!(
14297 out.status.success(),
14298 "{}",
14299 String::from_utf8_lossy(&out.stderr)
14300 );
14301 };
14302 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14303 g(&["checkout", "-b", "historical"]);
14304 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14305 g(&["add", "."]);
14306 g(&["commit", "-m", "old alternative"]);
14307 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14308 g(&["checkout", "-b", "magi/task-selected", &base]);
14309 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14310 g(&["add", "."]);
14311 g(&["commit", "-m", "measurement infrastructure"]);
14312 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14313 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14314 g(&["add", "."]);
14315 g(&["commit", "-m", "dependent implementation"]);
14316 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14317 g(&["checkout", "main"]);
14318 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14319 g(&["add", "."]);
14320 g(&["commit", "-m", "unrelated base change"]);
14321 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14322 let origin = tmp.path().join("origin.git");
14323 g(&[
14324 "clone",
14325 "--bare",
14326 repo.to_str().unwrap(),
14327 origin.to_str().unwrap(),
14328 ]);
14329 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14330 g(&["fetch", "origin"]);
14331 let selection = crate::pr_recovery::Selection {
14332 url: "https://example.test/project/pull/1".into(),
14333 branch: "magi/task-selected".into(),
14334 head: head.clone(),
14335 source_head: None,
14336 source_run: None,
14337 source_proof: None,
14338 disposition: Some(crate::pr_recovery::Disposition::Review),
14339 instruction: "preserve measured behavior".into(),
14340 approved_at: jiff::Timestamp::now(),
14341 };
14342 let mut template = runner_at(RunStatus::Reviewing);
14343 template.state.repo = repo.clone();
14344 template.state.base_commit = current.clone();
14345 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14346 template.state.origin = Some(crate::run::Origin::queue("selected"));
14347 template.state.pr_head = Some(selection.branch.clone());
14348 template.state.pr_recovery = Some(selection.clone());
14349 let mut runner =
14350 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14351 .await
14352 .unwrap();
14353 let wt = runner.state.winner().unwrap().worktree.clone();
14354 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14355 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14356 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14357 assert!(runner.state.seeds.is_empty());
14358 assert_eq!(runner.state.instruction, "preserve measured behavior");
14359 runner.sync_to_base().await.unwrap();
14360 assert_eq!(
14361 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14362 "prerequisite"
14363 );
14364 assert_eq!(
14365 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14366 "measured implementation"
14367 );
14368 assert_eq!(
14369 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14370 "current main change"
14371 );
14372 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14373 assert_eq!(
14374 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14375 head
14376 );
14377
14378 let mut partial = runner_at(RunStatus::Reviewing);
14380 partial.state.repo = repo.clone();
14381 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14382 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14383 run: Some("exit 1".into()),
14384 ..Default::default()
14385 }];
14386 let selection = runner.state.pr_recovery.clone().unwrap();
14387 partial.state.pr_recovery = Some(selection.clone());
14388 partial.state.pr_head = Some(selection.branch.clone());
14389 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14390 let id = partial.state.id.clone();
14391 let error = match Runner::open_selected(
14392 &repo,
14393 partial.state,
14394 partial.roles,
14395 1,
14396 runner.state.base_commit.clone(),
14397 selection,
14398 )
14399 .await
14400 {
14401 Ok(_) => panic!("setup must fail"),
14402 Err(e) => e,
14403 };
14404 assert_eq!(
14405 error
14406 .downcast_ref::<crate::run::StartupFailure>()
14407 .unwrap()
14408 .run,
14409 id
14410 );
14411 let failed = RunState::load(&id).unwrap();
14412 assert_eq!(failed.status, RunStatus::Failed);
14413 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14414 assert!(Runner::resume(&id).is_err());
14415 runner.state.status = RunStatus::Ready;
14416 runner.state.driver_exited = true;
14417 runner.state.save().unwrap();
14418 let mut retry = runner_at(RunStatus::Reviewing);
14419 retry.state.repo = repo.clone();
14420 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14421 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14422 reselected.approved_at = jiff::Timestamp::now();
14423 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14424 retry.state.pr_head = Some(reselected.branch.clone());
14425 retry.state.pr_recovery = Some(reselected.clone());
14426 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14427 .await
14428 .unwrap();
14429 let restarted = Runner::open_selected(
14430 &repo,
14431 retry.state,
14432 retry.roles,
14433 1,
14434 runner.state.base_commit.clone(),
14435 reselected,
14436 )
14437 .await
14438 .unwrap();
14439 assert_eq!(
14440 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14441 .await
14442 .unwrap(),
14443 head
14444 );
14445 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14446 assert_eq!(
14447 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14448 head
14449 );
14450 }
14451
14452 #[tokio::test]
14453 #[cfg(unix)]
14454 async fn fallback_and_interrupted_restoration_require_coordination() {
14455 use std::os::fd::AsRawFd as _;
14456 ask_test_home();
14457 let tmp = tempfile::tempdir().unwrap();
14458 let mut runner = registered_runner(tmp.path()).await;
14459 runner.state.config.graph.timeout_resource_wait = 1;
14460 let winner = runner.state.candidates[0].clone();
14461 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14462 let leftover = winner.worktree.join("leftover");
14463 std::fs::write(&leftover, "untrusted edits").unwrap();
14464 let next = tmp.path().join("next-fixer");
14465 let mut failed = spec("first");
14466 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14467 let mut successor = spec("second");
14468 successor.command = vec![
14469 "sh".into(),
14470 "-c".into(),
14471 format!("touch '{}'; echo done", next.display()),
14472 ];
14473 let attempts = vec![
14474 (failed.clone(), "first".into()),
14475 (successor.clone(), "second".into()),
14476 ];
14477 let lock_path = runner.state.repo.join(".git/updater.lock");
14478 let external = std::fs::File::create(&lock_path).unwrap();
14479 assert_eq!(
14480 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14481 0
14482 );
14483 let artifacts = tmp.path().join("artifacts");
14484 let build = |spec, seat| SeatJob {
14485 spec,
14486 seat,
14487 cwd: winner.worktree.clone(),
14488 prompt: "fixture".into(),
14489 timeout: Duration::from_secs(10),
14490 allow_write: true,
14491 sessions: false,
14492 artifacts: artifacts.clone(),
14493 stem: "fix".into(),
14494 handover: None,
14495 };
14496 let (_, _, out) = runner
14497 .ask_fixer_among(
14498 attempts.clone(),
14499 "graft",
14500 None,
14501 Some((&winner.worktree, &head)),
14502 build,
14503 )
14504 .await;
14505 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14506 assert!(leftover.exists(), "contention must prevent clean");
14507 assert!(!next.exists());
14508 assert!(runner.state.handovers.is_empty());
14509 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14511 from_run: "earlier".into(),
14512 generation: 1,
14513 mode: crate::recovery::Mode::VerifyRepair,
14514 reasons: Vec::new(),
14515 reviewed_head: None,
14516 rounds: Vec::new(),
14517 open_findings: Vec::new(),
14518 evidence: None,
14519 criteria_inherited: true,
14520 repair: crate::recovery::Repair::Started {
14521 before: head.clone(),
14522 },
14523 });
14524 assert!(
14525 runner
14526 .recovery_repair(&winner)
14527 .await
14528 .unwrap_err()
14529 .is::<crate::repo_lock::ResourceBlocked>()
14530 );
14531 assert!(matches!(
14532 runner.state.recovery.as_ref().unwrap().repair,
14533 crate::recovery::Repair::Started { .. }
14534 ));
14535 assert!(leftover.exists());
14536 drop(external);
14537 runner.recovery_repair(&winner).await.unwrap();
14538 assert!(!leftover.exists());
14539 assert!(matches!(
14540 runner.state.recovery.as_ref().unwrap().repair,
14541 crate::recovery::Repair::Done { .. }
14542 ));
14543 std::fs::remove_file(&lock_path).unwrap();
14545 std::fs::create_dir(&lock_path).unwrap();
14546 std::fs::write(&leftover, "untrusted edits").unwrap();
14547 let (_, _, out) = runner
14548 .ask_fixer_among(
14549 attempts,
14550 "graft",
14551 None,
14552 Some((&winner.worktree, &head)),
14553 build,
14554 )
14555 .await;
14556 assert!(matches!(out, AgentOutcome::Failed(_)));
14557 assert!(leftover.exists());
14558 assert!(!next.exists());
14559 assert_eq!(
14560 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14561 head
14562 );
14563 }
14564
14565 #[cfg(unix)]
14566 async fn registered_runner(root: &Path) -> Runner {
14567 repository_runner(root, crate::config::test_coordination()).await
14568 }
14569
14570 async fn repository_runner(
14571 root: &Path,
14572 coordination: Option<crate::config::Coordination>,
14573 ) -> Runner {
14574 ask_test_home();
14575 let repo = root.join("repo");
14576 std::fs::create_dir_all(&repo).unwrap();
14577 init_repo(&repo);
14578 if coordination.is_some() {
14579 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14580 }
14581 let mut runner = runner_at(RunStatus::Prep);
14582 runner.state.repo = repo.clone();
14583 runner.state.config.coordination = coordination;
14584 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14585 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14586 runner.state.config.graph.advise = false;
14587 runner.roles.implementers = vec![spec("implement")];
14588 runner.roles.judges = vec![spec("judge")];
14589 runner.prep().await.unwrap();
14590 runner
14591 }
14592
14593 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14594
14595 #[tokio::test]
14596 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14597 let tmp = tempfile::tempdir().unwrap();
14598 let mut runner = repository_runner(tmp.path(), None).await;
14599 let prompts = runner.state.config.prompts.clone();
14600 let run = runner.state.id.clone();
14601 let ctx = WaveCtx {
14602 carry_seats: false,
14603 pauses: None,
14604 run: &run,
14605 node: "advise",
14606 prompts: &prompts,
14607 cache: None,
14608 round: None,
14609 };
14610 let jobs: Vec<_> = ["alpha", "beta"]
14611 .into_iter()
14612 .map(|id| SeatJob {
14613 spec: scripted(id, AUTH_OUTAGE),
14614 seat: SeatState::new(id, id, 7),
14615 prompt: "propose".into(),
14616 cwd: runner.state.candidates[0].worktree.clone(),
14617 timeout: Duration::from_secs(10),
14618 allow_write: false,
14619 sessions: false,
14620 artifacts: agent::artifacts_dir(&runner.state.dir()),
14621 stem: id.into(),
14622 handover: None,
14623 })
14624 .collect();
14625 let mut losses = Vec::new();
14626 let results = ask_wave_with(
14627 jobs.clone(),
14628 runner.sem.clone(),
14629 2,
14630 &[],
14631 &ctx,
14632 &mut losses,
14633 &mut runner.state,
14634 &|_: &str| Ok(()),
14635 )
14636 .await;
14637 assert!(
14638 results
14639 .iter()
14640 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14641 );
14642 assert_eq!(runner.state.quota.len(), 2);
14643 assert!(
14644 runner
14645 .state
14646 .quota
14647 .iter()
14648 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14649 );
14650 assert!(!runner.state.advise_attempted);
14651 runner.stop_for_resource().unwrap();
14652 assert_eq!(runner.state.status, RunStatus::Blocked);
14653 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14654 assert_eq!(
14655 runner.state.resource_block.as_ref().unwrap().kind,
14656 "agent authentication"
14657 );
14658 runner.state.resource_block = None;
14661 runner.state.quota.clear();
14662 let mut mixed = jobs;
14663 mixed[1].spec.command = vec!["false".into()];
14664 let results = ask_wave_with(
14665 mixed,
14666 runner.sem.clone(),
14667 0,
14668 &[],
14669 &ctx,
14670 &mut losses,
14671 &mut runner.state,
14672 &|_: &str| Ok(()),
14673 )
14674 .await;
14675 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14676 assert!(runner.state.resource_block.is_none());
14677 assert_eq!(losses.len(), 1);
14678 }
14679
14680 #[tokio::test]
14681 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14682 let tmp = tempfile::tempdir().unwrap();
14683 let mut runner = repository_runner(tmp.path(), None).await;
14684 let implementer = scripted("implement", AUTH_OUTAGE);
14685 runner.roles.implementers = vec![implementer.clone()];
14686 runner.state.config.agents = vec![implementer];
14687 runner.implement().await.unwrap();
14688 assert_eq!(runner.state.status, RunStatus::Blocked);
14689 assert_eq!(runner.state.quota.len(), 1);
14690 assert_eq!(
14691 runner.state.quota[0].cause,
14692 agent::QuotaCause::Authentication
14693 );
14694 let candidate = &runner.state.candidates[0];
14695 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14696 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14697 }
14698
14699 #[tokio::test]
14700 #[cfg(unix)]
14701 async fn parked_implementer_persists_completed_work() {
14702 for registered in [false, true] {
14703 let tmp = tempfile::tempdir().unwrap();
14704 let mut runner = registered_runner(tmp.path()).await;
14705 if !registered {
14706 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14707 }
14708 let marker = tmp.path().join("started");
14709 let release = tmp.path().join("release");
14710 let mut implementer = spec("implement");
14711 implementer.command = vec![
14712 "sh".into(),
14713 "-c".into(),
14714 format!(
14715 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14716 marker.display(),
14717 release.display()
14718 ),
14719 ];
14720 runner.roles.implementers = vec![implementer.clone()];
14721 runner.state.config.agents = vec![implementer];
14722 let pause = runner.pause.clone();
14723 let (result, ()) = tokio::join!(runner.implement(), async {
14724 tokio::time::timeout(Duration::from_secs(10), async {
14725 while !marker.exists() {
14726 tokio::time::sleep(Duration::from_millis(10)).await;
14727 }
14728 })
14729 .await
14730 .unwrap();
14731 pause.park();
14732 std::fs::write(&release, "go").unwrap();
14733 });
14734 result.unwrap();
14735 let candidate = &runner.state.candidates[0];
14736 assert!(candidate.commits > 0, "{candidate:?}");
14737 assert!(candidate.summary.contains("saved-result"));
14738 assert_eq!(candidate.files, 1);
14739 let before = candidate.commits;
14740 runner.implement().await.unwrap();
14741 assert_eq!(runner.state.candidates[0].commits, before);
14742 }
14743 }
14744
14745 #[tokio::test]
14746 #[cfg(unix)]
14747 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14748 let tmp = tempfile::tempdir().unwrap();
14749 let mut runner = registered_runner(tmp.path()).await;
14750 let winner = runner.state.candidates[0].clone();
14751 let cache = tmp.path().join("cache");
14752 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14753 runner.state.config.graph.timeout_resource_wait = 0;
14754 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14755 let crate::cache::AcquireOutcome::Acquired(held) =
14756 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14757 else {
14758 panic!("cache must be free")
14759 };
14760 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14761 from_run: "earlier".into(),
14762 generation: 1,
14763 mode: crate::recovery::Mode::VerifyRepair,
14764 reasons: Vec::new(),
14765 reviewed_head: None,
14766 rounds: Vec::new(),
14767 open_findings: Vec::new(),
14768 evidence: None,
14769 criteria_inherited: true,
14770 repair: crate::recovery::Repair::NotStarted,
14771 });
14772 runner.recovery_repair(&winner).await.unwrap();
14773 assert!(matches!(
14774 runner.state.recovery.as_ref().unwrap().repair,
14775 crate::recovery::Repair::WaitingForRestore { .. }
14776 ));
14777 drop(held);
14778 runner.state.resource_block = None;
14779 runner.state.config.graph.timeout_resource_wait = 1;
14780 runner.recovery_repair(&winner).await.unwrap();
14781 assert!(matches!(
14782 runner.state.recovery.as_ref().unwrap().repair,
14783 crate::recovery::Repair::Done { .. }
14784 ));
14785 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14786 }
14787
14788 #[tokio::test]
14789 #[cfg(unix)]
14790 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14791 use std::os::fd::AsRawFd as _;
14792 for graft in [false, true] {
14793 let tmp = tempfile::tempdir().unwrap();
14794 let mut runner = registered_runner(tmp.path()).await;
14795 runner.state.config.graph.timeout_resource_wait = 1;
14796 let winner = runner.state.candidates[0].clone();
14797 let marker = tmp.path().join("fixer-started");
14798 let mut fixer = spec("fixer");
14799 fixer.command = vec![
14800 "sh".into(),
14801 "-c".into(),
14802 format!("echo started >> '{}'; echo done", marker.display()),
14803 ];
14804 runner.roles.fixer = Some(vec![fixer]);
14805 if graft {
14806 let mut loser = winner.clone();
14807 loser.label = 'B';
14808 loser.index = 1;
14809 runner.state.candidates.push(loser);
14810 runner.state.config.graph.graft_runner_up = true;
14811 runner.state.tally = Some(Tally {
14812 first_choice: BTreeMap::new(),
14813 borda: BTreeMap::new(),
14814 winner: winner.label,
14815 rankings: 1,
14816 unanimous_initial: true,
14817 deliberated: false,
14818 changed_votes: 0,
14819 unanimous_final: true,
14820 tie_break: None,
14821 judges: 1,
14822 present: 1,
14823 quorum: 1,
14824 met_quorum: true,
14825 uncontested: None,
14826 });
14827 runner.state.judgements.push(crate::run::Judgement {
14828 judge: 1,
14829 seat: "judge-1".into(),
14830 agent: "judge".into(),
14831 ranking: vec![winner.label, 'B'],
14832 reasons: BTreeMap::new(),
14833 confidence: None,
14834 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14835 order: vec![0, 1],
14836 failed: None,
14837 duration_ms: 0,
14838 });
14839 } else {
14840 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14841 from_run: "earlier".into(),
14842 generation: 1,
14843 mode: crate::recovery::Mode::VerifyRepair,
14844 reasons: Vec::new(),
14845 reviewed_head: None,
14846 rounds: Vec::new(),
14847 open_findings: Vec::new(),
14848 evidence: None,
14849 criteria_inherited: true,
14850 repair: crate::recovery::Repair::NotStarted,
14851 });
14852 }
14853 let reader = crate::repo_lock::SeatLease::for_checkout(
14854 &winner.worktree,
14855 false,
14856 crate::config::test_coordination().as_ref(),
14857 )
14858 .await
14859 .unwrap();
14860 let pause = runner.pause.clone();
14861 let (result, ()) = tokio::join!(
14862 async {
14863 if graft {
14864 runner.graft().await
14865 } else {
14866 runner.recovery_repair(&winner).await
14867 }
14868 },
14869 async {
14870 tokio::time::sleep(Duration::from_millis(100)).await;
14871 pause.park();
14872 }
14873 );
14874 result.unwrap();
14875 assert!(!marker.exists(), "a blocked seat must not spawn");
14876 runner.state = RunState::load(&runner.state.id).unwrap();
14878 if graft {
14879 assert_eq!(
14880 runner.state.graft.as_ref().unwrap().outcome,
14881 GraftOutcome::WaitingForRestore
14882 );
14883 } else {
14884 assert!(matches!(
14885 runner.state.recovery.as_ref().unwrap().repair,
14886 crate::recovery::Repair::WaitingForRestore { .. }
14887 ));
14888 }
14889 drop(reader);
14890 runner.pause = Pause::new();
14891 runner.state.resource_block = None;
14892 let external =
14894 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14895 assert_eq!(
14896 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14897 0
14898 );
14899 let leftover = winner.worktree.join("leftover");
14900 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14901 let result = if graft {
14902 runner.graft().await
14903 } else {
14904 runner.recovery_repair(&winner).await
14905 };
14906 assert!(
14907 result
14908 .unwrap_err()
14909 .is::<crate::repo_lock::ResourceBlocked>()
14910 );
14911 assert!(leftover.exists());
14912 assert!(!marker.exists());
14913 drop(external);
14914 runner.state.resource_block = None;
14915 let result = if graft {
14916 runner.graft().await
14917 } else {
14918 runner.recovery_repair(&winner).await
14919 };
14920 result.unwrap();
14921 assert!(!leftover.exists(), "restore must precede retry");
14922 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14923 if graft {
14924 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14925 } else {
14926 assert!(matches!(
14927 runner.state.recovery.as_ref().unwrap().repair,
14928 crate::recovery::Repair::Done { .. }
14929 ));
14930 }
14931 }
14932 }
14933
14934 #[tokio::test]
14935 #[cfg(unix)]
14936 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14937 use std::os::fd::AsRawFd as _;
14938 let tmp = tempfile::tempdir().unwrap();
14939 let mut runner = registered_runner(tmp.path()).await;
14940 runner.state.config.graph.timeout_resource_wait = 1;
14941 let winner = runner.state.candidates[0].clone();
14942 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14943 runner.state.tally = Some(Tally {
14944 first_choice: BTreeMap::new(),
14945 borda: BTreeMap::new(),
14946 winner: winner.label,
14947 rankings: 0,
14948 unanimous_initial: false,
14949 deliberated: false,
14950 changed_votes: 0,
14951 unanimous_final: false,
14952 tie_break: None,
14953 judges: 0,
14954 present: 0,
14955 quorum: 0,
14956 met_quorum: true,
14957 uncontested: Some("fixture".into()),
14958 });
14959 runner.state.graft = Some(GraftRecord {
14960 pre: pre.clone(),
14961 post: None,
14962 lines: None,
14963 limit: None,
14964 agent: None,
14965 outcome: GraftOutcome::Running,
14966 requested: Vec::new(),
14967 adoption: None,
14968 adoption_error: None,
14969 });
14970 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14971 git::commit_all(&winner.worktree, "unfinished graft")
14972 .await
14973 .unwrap();
14974 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14975 let leftover = winner.worktree.join("leftover");
14976 std::fs::write(&leftover, "untrusted edits").unwrap();
14977 let path = runner.state.repo.join(".git/updater.lock");
14978 let external = std::fs::File::create(&path).unwrap();
14979 assert_eq!(
14980 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14981 0
14982 );
14983 assert!(
14984 runner
14985 .graft()
14986 .await
14987 .unwrap_err()
14988 .is::<crate::repo_lock::ResourceBlocked>()
14989 );
14990 assert!(matches!(
14991 runner.state.graft.as_ref().unwrap().outcome,
14992 GraftOutcome::Running
14993 ));
14994 assert_eq!(
14995 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14996 unfinished
14997 );
14998 assert!(leftover.exists());
14999 drop(external);
15000 std::fs::remove_file(&path).unwrap();
15001 std::fs::create_dir(&path).unwrap();
15002 assert!(runner.graft().await.is_err());
15003 assert!(matches!(
15004 runner.state.graft.as_ref().unwrap().outcome,
15005 GraftOutcome::Running
15006 ));
15007 assert_eq!(
15008 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15009 unfinished
15010 );
15011 assert!(leftover.exists());
15012 std::fs::remove_dir(&path).unwrap();
15013 runner.graft().await.unwrap();
15014 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15015 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15016 assert!(!leftover.exists());
15017 }
15018
15019 #[tokio::test]
15020 #[cfg(unix)]
15021 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15022 use std::os::fd::AsRawFd as _;
15023 let tmp = tempfile::tempdir().unwrap();
15024 let mut runner = registered_runner(tmp.path()).await;
15025 runner.state.prep_complete = Some(false);
15026 runner.state.config.graph.timeout_resource_wait = 1;
15027 runner.state.save().unwrap();
15028 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15029 assert_eq!(
15030 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15031 0
15032 );
15033 runner.execute().await.unwrap();
15034 let disk = RunState::load(&runner.state.id).unwrap();
15035 assert!(disk.driver_exited);
15036 assert_eq!(disk.driver_pid, None);
15037 assert_eq!(runner.state.status, RunStatus::Blocked);
15038 assert_eq!(
15039 runner.state.resource_block.as_ref().unwrap().kind,
15040 "repository coordination"
15041 );
15042 assert!(runner.state.prep_failure.is_none());
15043 assert!(runner.state.seats.is_empty());
15044 runner.pause.park();
15045 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15048 .await
15049 .unwrap()
15050 .unwrap();
15051 assert!(runner.state.parked);
15052 assert!(runner.state.prep_failure.is_none());
15053 let disk = RunState::load(&runner.state.id).unwrap();
15054 assert!(disk.driver_exited);
15055 assert_eq!(disk.driver_pid, None);
15056 }
15057
15058 #[tokio::test]
15059 #[cfg(unix)]
15060 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15061 use std::os::fd::AsRawFd as _;
15062 let tmp = tempfile::tempdir().unwrap();
15063 let mut runner = registered_runner(tmp.path()).await;
15064 runner.state.prep_complete = Some(false);
15065 runner.state.config.graph.timeout_resource_wait = 30;
15066 runner.state.save().unwrap();
15067 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15068 assert_eq!(
15069 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15070 0
15071 );
15072 assert!(
15073 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15074 .await
15075 .is_err()
15076 );
15077 let disk = RunState::load(&runner.state.id).unwrap();
15078 assert!(disk.driver_exited);
15079 assert_eq!(disk.driver_pid, None);
15080 assert!(disk.status.resumable());
15081 assert!(runner.state.driver_exited);
15082 assert_eq!(runner.state.driver_pid, None);
15083 runner.state.save().unwrap();
15084 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15085 }
15086
15087 #[tokio::test]
15088 #[cfg(unix)]
15089 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15090 let tmp = tempfile::tempdir().unwrap();
15091 let mut runner = registered_runner(tmp.path()).await;
15092 let winner = runner.state.candidates[0].clone();
15093 let cache = tmp.path().join("cache");
15094 let started = tmp.path().join("pre-gate-started");
15095 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15096 runner.state.config.verify.gate =
15097 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15098 let home = crate::run::home();
15099 let owner =
15100 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15101 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15102 crate::cache::AcquireOutcome::Acquired(g) => g,
15103 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15104 };
15105 let mut gate = Box::pin(runner.run_gate(&winner));
15106 assert!(
15107 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15108 .await
15109 .is_err()
15110 );
15111 assert!(!started.exists(), "pre-gate must wait for the cache too");
15112 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15116 let editor = crate::repo_lock::SeatLease::checkout_with(
15117 &winner.worktree,
15118 true,
15119 crate::config::test_coordination().as_ref(),
15120 &wait,
15121 )
15122 .await
15123 .unwrap();
15124 drop(editor);
15125 guard.release();
15126 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15127 .await
15128 .unwrap()
15129 .unwrap();
15130 drop(gate);
15131 assert!(outcomes.iter().all(CommandOutcome::ok));
15132 assert!(started.exists());
15133 }
15134
15135 #[tokio::test]
15136 #[cfg(unix)]
15137 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15138 use std::os::fd::AsRawFd as _;
15139
15140 ask_test_home();
15141 let tmp = tempfile::tempdir().unwrap();
15142 let repo = tmp.path().join("repo");
15143 std::fs::create_dir_all(&repo).unwrap();
15144 init_repo(&repo);
15145 let common = repo.join(".git");
15146 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15147 let mut runner = runner_at(RunStatus::Prep);
15148 runner.state.repo = repo.clone();
15149 runner.state.config.coordination = crate::config::test_coordination();
15150 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15151 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15152 runner.state.config.graph.advise = false;
15153 runner.roles.implementers = vec![spec("implement")];
15154 runner.roles.judges = vec![spec("judge")];
15155 runner.prep().await.unwrap();
15156 let winner = runner.state.candidates[0].clone();
15157 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15160 let completed = tmp.path().join("seat-completed");
15161 runner.roles.implementers[0].command = vec![
15162 "sh".into(),
15163 "-c".into(),
15164 format!("touch '{}'; echo completed", completed.display()),
15165 ];
15166 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15167 assert_eq!(
15168 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15169 0
15170 );
15171 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15172 let mut implement = Box::pin(runner.implement());
15173 tokio::select! {
15174 result = &mut implement => panic!("rescue must wait: {result:?}"),
15175 result = tokio::time::timeout(Duration::from_secs(10), async {
15176 while !completed.exists() {
15177 tokio::time::sleep(Duration::from_millis(20)).await;
15178 }
15179 }) => result.unwrap(),
15180 }
15181 assert!(
15182 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15183 .await
15184 .is_err()
15185 );
15186 assert_eq!(
15187 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15188 before
15189 );
15190 assert_eq!(
15191 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15192 0
15193 );
15194 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15195 .await
15196 .unwrap()
15197 .unwrap();
15198 drop(implement);
15199 assert_ne!(
15200 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15201 before
15202 );
15203 assert!(git::is_clean(&winner.worktree).await.unwrap());
15204
15205 let started = tmp.path().join("pre-gate-started");
15206 let proceed = tmp.path().join("pre-gate-proceed");
15207 let checked = tmp.path().join("gate-ran");
15208 runner.state.config.verify.pre_gate = vec![format!(
15209 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15210 started.display(),
15211 proceed.display()
15212 )];
15213 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15214 assert_eq!(
15215 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15216 0
15217 );
15218 let mut gate = Box::pin(runner.run_gate(&winner));
15219 assert!(
15220 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15221 .await
15222 .is_err()
15223 );
15224 assert!(!started.exists());
15225 assert!(!checked.exists());
15226 assert!(git::is_clean(&winner.worktree).await.unwrap());
15227 assert_eq!(
15228 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15229 0
15230 );
15231 tokio::select! {
15232 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15233 result = tokio::time::timeout(Duration::from_secs(10), async {
15234 while !started.exists() {
15235 tokio::time::sleep(Duration::from_millis(20)).await;
15236 }
15237 }) => result.unwrap(),
15238 }
15239 assert_eq!(
15240 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15241 -1,
15242 "mechanical edits must hold the real lock"
15243 );
15244 std::fs::write(proceed, "ready").unwrap();
15245 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15246 .await
15247 .unwrap()
15248 .unwrap();
15249 drop(gate);
15250 assert!(outcomes.iter().all(CommandOutcome::ok));
15251 assert!(checked.exists());
15252 assert!(runner.state.pre_gate_commit.is_some());
15253 assert!(git::is_clean(&winner.worktree).await.unwrap());
15254 assert_eq!(
15255 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15256 0,
15257 "gate completion must release coordination"
15258 );
15259 drop(outsider);
15260 std::fs::remove_file(common.join("updater.lock")).unwrap();
15261 std::fs::create_dir(common.join("updater.lock")).unwrap();
15262 std::fs::remove_file(&started).unwrap();
15263 std::fs::remove_file(&checked).unwrap();
15264 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15265 let error = runner.run_gate(&winner).await.unwrap_err();
15266 assert!(
15267 error
15268 .to_string()
15269 .contains("acquire mandatory repository lock")
15270 );
15271 assert!(
15272 !started.exists(),
15273 "failed acquisition must prevent pre-gate"
15274 );
15275 assert!(!checked.exists(), "failed acquisition must prevent gate");
15276 assert!(git::is_clean(&winner.worktree).await.unwrap());
15277 assert_eq!(
15278 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15279 head
15280 );
15281 }
15282
15283 #[tokio::test]
15284 #[cfg(unix)]
15285 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15286 use std::os::fd::AsRawFd as _;
15287 use std::time::Duration;
15288
15289 ask_test_home();
15290 let tmp = tempfile::tempdir().unwrap();
15291 let repo = tmp.path().join("repo");
15292 std::fs::create_dir_all(&repo).unwrap();
15293 init_repo(&repo);
15294 let common = repo.join(".git");
15295 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15296 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15297 assert_eq!(
15298 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15299 0
15300 );
15301 let mut runner = runner_at(RunStatus::Prep);
15302 runner.state.repo = repo.clone();
15303 runner.state.config.coordination = crate::config::test_coordination();
15304 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15305 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15306 runner.state.config.graph.advise = true;
15307 runner.state.config.graph.advisors = 1;
15308 runner.roles.implementers = vec![spec("implement")];
15309 runner.roles.judges = vec![spec("judge")];
15310 let root = runner.state.worktree_root();
15311 let started = tmp.path().join("setup-started");
15312 let proceed = tmp.path().join("setup-proceed");
15313 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15314 run: Some(format!(
15315 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15316 started.display(),
15317 proceed.display()
15318 )),
15319 ..Default::default()
15320 }];
15321 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15322 let config = git::git(&repo, &["config", "--local", "--list"])
15323 .await
15324 .unwrap();
15325 let mut prep = Box::pin(runner.prep());
15326 assert!(
15327 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15328 .await
15329 .is_err()
15330 );
15331 assert!(
15332 !root.exists(),
15333 "contention must prevent even checkout creation"
15334 );
15335 assert!(!started.exists(), "contention must prevent setup");
15336 assert_eq!(
15337 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15338 branches
15339 );
15340 assert_eq!(
15341 git::git(&repo, &["config", "--local", "--list"])
15342 .await
15343 .unwrap(),
15344 config
15345 );
15346 drop(lock);
15347 tokio::select! {
15348 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15349 result = tokio::time::timeout(Duration::from_secs(10), async {
15350 while !started.exists() {
15351 tokio::time::sleep(Duration::from_millis(20)).await;
15352 }
15353 }) => result.unwrap(),
15354 }
15355 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15356 assert_eq!(
15357 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15358 -1
15359 );
15360 assert!(
15361 tokio::time::timeout(
15362 Duration::from_millis(100),
15363 crate::repo_lock::SeatLease::for_checkout(
15364 &repo,
15365 true,
15366 crate::config::test_coordination().as_ref()
15367 )
15368 )
15369 .await
15370 .is_err(),
15371 "preparation must also exclude local editors"
15372 );
15373 std::fs::write(proceed, "ready").unwrap();
15374 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15375 .await
15376 .unwrap()
15377 .unwrap();
15378 drop(prep);
15379 assert_eq!(runner.state.prep_complete, Some(true));
15380 assert!(root.join("judge-1").exists());
15381 assert!(root.join("advisor-1").exists());
15382 assert_eq!(
15383 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15384 0
15385 );
15386 }
15387
15388 #[tokio::test]
15389 #[cfg(unix)]
15390 async fn preparation_lock_failure_prevents_repository_changes() {
15391 ask_test_home();
15392 let tmp = tempfile::tempdir().unwrap();
15393 let repo = tmp.path().join("repo");
15394 std::fs::create_dir_all(&repo).unwrap();
15395 init_repo(&repo);
15396 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15397 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15400 let mut runner = runner_at(RunStatus::Prep);
15401 runner.state.repo = repo.clone();
15402 runner.state.config.coordination = crate::config::test_coordination();
15403 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15404 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15405 runner.roles.implementers = vec![spec("implement")];
15406 let error = runner.prep().await.unwrap_err();
15407 assert!(
15408 error
15409 .to_string()
15410 .contains("acquire mandatory repository lock")
15411 );
15412 assert!(!runner.state.worktree_root().exists());
15413 assert!(runner.state.candidates.is_empty());
15414 assert!(!repo.join(".git/config.worktree").exists());
15415 assert_eq!(
15416 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15417 "* main"
15418 );
15419 }
15420
15421 #[tokio::test]
15422 async fn preparation_records_creation_intent_before_git_can_fail() {
15423 ask_test_home();
15424 let tmp = tempfile::tempdir().unwrap();
15425 let repo = tmp.path().join("repo");
15426 std::fs::create_dir_all(&repo).unwrap();
15427 init_repo(&repo);
15428 let mut runner = runner_at(RunStatus::Prep);
15429 runner.state.repo = repo.clone();
15430 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15431 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15432 runner.state.config.blind.commit_msg_hook = false;
15433 runner.roles.implementers = vec![spec("one")];
15434 let label = blind::assign_labels(1, runner.state.seed)[0];
15435 let branch = runner.state.branch_for(label);
15436 git::git(&repo, &["branch", &branch]).await.unwrap();
15439 assert!(runner.prep().await.is_err());
15440 let saved = RunState::load(&runner.state.id).unwrap();
15441 assert!(saved.candidates.is_empty());
15442 let event = saved
15443 .events
15444 .iter()
15445 .find(|e| e.node == "prep-intent")
15446 .unwrap();
15447 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15448 assert_eq!(intent["branch"], branch);
15449 assert_eq!(intent["start"], runner.state.base_commit);
15450 assert_eq!(
15451 intent["worktree"],
15452 runner
15453 .state
15454 .worktree_root()
15455 .join(format!("cand-{label}"))
15456 .to_string_lossy()
15457 .as_ref()
15458 );
15459 assert_eq!(intent["picks"], serde_json::json!([]));
15460 }
15461
15462 #[tokio::test]
15463 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15464 ask_test_home();
15465 let tmp = tempfile::tempdir().unwrap();
15466 let repo = tmp.path().join("repo");
15467 std::fs::create_dir_all(&repo).unwrap();
15468 init_repo(&repo);
15469 let mut runner = runner_at(RunStatus::Prep);
15470 runner.state.repo = repo.clone();
15471 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15472 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15473 runner.state.sources.pick_commits.push(refs::Input {
15474 token: "missing".into(),
15475 sha: "0".repeat(40),
15476 provenance: refs::Provenance::ExplicitCli,
15477 });
15478 assert!(runner.prep().await.is_err());
15479 assert!(runner.state.candidates.is_empty());
15480 assert!(!runner.state.worktree_root().exists());
15481 }
15482
15483 #[tokio::test]
15484 async fn prose_and_legacy_inference_never_import_into_candidates() {
15485 ask_test_home();
15486 let tmp = tempfile::tempdir().unwrap();
15487 let repo = tmp.path().join("repo");
15488 std::fs::create_dir_all(&repo).unwrap();
15489 init_repo(&repo);
15490 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15491 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15492 .await
15493 .unwrap();
15494 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15495 git::git(&repo, &["add", "."]).await.unwrap();
15496 git::git(&repo, &["commit", "-m", "historical implementation"])
15497 .await
15498 .unwrap();
15499 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15500 git::git(&repo, &["checkout", "main"]).await.unwrap();
15501 for instruction in [
15502 format!("Example: magi/history/A at {source}"),
15503 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15504 format!("Never import magi/history/A or {source}"),
15505 format!("Conversation answer: I mentioned {source} only as background"),
15506 ] {
15507 let mut runner = runner_at(RunStatus::Prep);
15508 runner.state.repo = repo.clone();
15509 runner.state.base_commit = base.clone();
15510 runner.state.instruction = instruction.clone();
15511 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15512 runner.state.config.blind.commit_msg_hook = false;
15513 runner.state.seeds = vec![refs::Seed {
15514 token: "magi/history/A".into(),
15515 sha: source.clone(),
15516 kind: refs::SeedKind::Unmerged,
15517 branch: true,
15518 detail: String::new(),
15519 }];
15520 runner.roles.implementers = vec![spec("one")];
15521 assert_eq!(seeded_instruction(&runner.state), instruction);
15522 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15524 legacy.as_object_mut().unwrap().remove("sources");
15525 runner.state = serde_json::from_value(legacy).unwrap();
15526 assert_eq!(runner.state.sources, refs::Sources::default());
15527 runner.state.save().unwrap();
15528 runner.state = RunState::load(&runner.state.id).unwrap();
15529 runner.prep().await.unwrap();
15530 for candidate in &runner.state.candidates {
15531 assert_eq!(
15532 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15533 base
15534 );
15535 assert!(!candidate.worktree.join("unwanted").exists());
15536 }
15537 }
15538 }
15539
15540 #[tokio::test]
15541 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15542 ask_test_home();
15543 let tmp = tempfile::tempdir().unwrap();
15544 let repo = tmp.path().join("repo");
15545 std::fs::create_dir_all(&repo).unwrap();
15546 init_repo(&repo);
15547 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15548 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15549 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15550 git::git(&repo, &["add", "."]).await.unwrap();
15551 git::git(&repo, &["commit", "-m", "required seed"])
15552 .await
15553 .unwrap();
15554 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15555 git::git(&repo, &["checkout", "main"]).await.unwrap();
15556 let mut runner = runner_at(RunStatus::Prep);
15557 runner.state.repo = repo.clone();
15558 runner.state.base_commit = base.clone();
15559 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15560 runner.state.config.graph.advise = false;
15561 runner.state.config.blind.commit_msg_hook = false;
15562 runner.state.sources =
15563 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15564 .await
15565 .unwrap();
15566 runner.roles.implementers = vec![spec("one"), spec("two")];
15567 runner.roles.judges = vec![spec("judge")];
15568 let label = blind::assign_labels(2, runner.state.seed)[0];
15569 let branch = runner.state.branch_for(label);
15570 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15571 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15572 .await
15573 .unwrap();
15574 runner.state.candidates.push(Candidate {
15575 index: 0,
15576 label,
15577 agent: "one".into(),
15578 branch,
15579 worktree,
15580 summary: String::new(),
15581 stat: String::new(),
15582 files: 0,
15583 commits: 0,
15584 empty: false,
15585 failed: None,
15586 verified_noop: None,
15587 duration_ms: 0,
15588 folded: false,
15589 });
15590 runner.state.save().unwrap();
15591 runner.state = RunState::load(&runner.state.id).unwrap();
15593 assert_eq!(runner.state.prep_complete, Some(false));
15594 runner.prep().await.unwrap();
15595 assert_eq!(runner.state.prep_complete, Some(true));
15596 assert_eq!(runner.state.candidates.len(), 2);
15597 for candidate in &runner.state.candidates {
15598 assert_eq!(
15599 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15600 git::tree_of(&repo, &seed).await.unwrap()
15601 );
15602 assert_eq!(
15603 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15604 "required commit"
15605 );
15606 }
15607 assert!(runner.state.worktree_root().join("judge-1").exists());
15608 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15609 runner.prep().await.unwrap();
15610 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15611
15612 runner.state.prep_complete = Some(false);
15613 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15614 run: Some(format!(
15615 "test -f '{}'",
15616 tmp.path().join("dependency-ready").display()
15617 )),
15618 ..Default::default()
15619 }];
15620 assert!(runner.prep().await.is_err());
15621 assert!(runner.state.prep_failure.is_some());
15622 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15623 runner.state = RunState::load(&runner.state.id).unwrap();
15624 assert!(
15625 runner.prep().await.is_err(),
15626 "correcting the cause alone must not auto-retry"
15627 );
15628 let failed_at = runner.state.events.last().unwrap().at;
15629 assert!(
15630 runner.retry_preparation(failed_at).is_err(),
15631 "old approvals cannot unlock the hold"
15632 );
15633 let mut task = crate::queue::Task::new(
15634 "retry".into(),
15635 "retry setup".into(),
15636 repo.clone(),
15637 crate::queue::Source::Human,
15638 );
15639 task.start(runner.state.id.clone());
15640 task.hold_machine(Some("setup failed".into()));
15641 task.release_operator("cli", None);
15642 runner
15643 .retry_preparation(task.resumption.as_ref().unwrap().at)
15644 .unwrap();
15645 runner.prep().await.unwrap();
15646 assert!(runner.state.prep_failure.is_none());
15647 assert_eq!(runner.state.prep_complete, Some(true));
15648 assert_eq!(runner.state.candidates.len(), 2);
15649 assert!(runner.state.worktree_root().join("judge-1").exists());
15650 for candidate in &runner.state.candidates {
15651 assert_eq!(
15652 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15653 git::tree_of(&repo, &seed).await.unwrap()
15654 );
15655 }
15656 }
15657
15658 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15659 let mut heads = Vec::new();
15660 for candidate in candidates {
15661 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15662 }
15663 heads
15664 }
15665
15666 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15667 ask_test_home();
15668 let tmp = tempfile::tempdir().unwrap();
15669 let repo = tmp.path().join("repo");
15670 std::fs::create_dir(&repo).unwrap();
15671 init_repo(&repo);
15672 let mut runner = runner_at(RunStatus::Reviewing);
15673 runner.state.repo = repo.clone();
15674 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15675 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15676 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15677 let branch = format!("candidate-{label}");
15678 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15679 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15680 .await
15681 .unwrap();
15682 if label == 'B' {
15683 std::fs::write(worktree.join("change"), "winner").unwrap();
15684 git::rescue_commit(&worktree, "candidate change")
15685 .await
15686 .unwrap();
15687 }
15688 runner.state.candidates.push(Candidate {
15689 index,
15690 label,
15691 agent: "alpha".into(),
15692 branch,
15693 worktree,
15694 summary: "completed reply".into(),
15695 stat: String::new(),
15696 files: usize::from(label == 'B'),
15697 commits: usize::from(label == 'B'),
15698 empty: label == 'A',
15699 failed: None,
15700 verified_noop: None,
15701 duration_ms: 1,
15702 folded: false,
15703 });
15704 }
15705 runner.state.judge_skipped = true;
15706 runner.state.tally = Some(Tally {
15707 first_choice: BTreeMap::new(),
15708 borda: BTreeMap::new(),
15709 winner: 'B',
15710 rankings: 0,
15711 unanimous_initial: false,
15712 deliberated: false,
15713 changed_votes: 0,
15714 unanimous_final: false,
15715 tie_break: None,
15716 judges: 0,
15717 present: 0,
15718 quorum: 0,
15719 met_quorum: true,
15720 uncontested: Some("only B produced a change".into()),
15721 });
15722 runner.state.save().unwrap();
15723 (tmp, runner)
15724 }
15725
15726 #[tokio::test]
15727 #[cfg(unix)]
15728 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15729 for (solo, failed_checkpoint) in
15730 [(true, false), (false, false), (true, true), (false, true)]
15731 {
15732 let (_tmp, mut runner) = preservation_resume_fixture().await;
15733 if solo {
15734 runner.state.candidates.retain(|c| c.label == 'A');
15735 }
15736 runner.state.candidates[0].empty = false;
15737 runner.state.candidates[0].summary.clear();
15738 runner.state.tally = None;
15739 runner.state.judge_skipped = false;
15740 runner.state.status = RunStatus::Implementing;
15741 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15742 let mut implementer = spec("alpha");
15743 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15744 runner.roles.implementers = vec![implementer];
15745 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15746 if failed_checkpoint {
15747 std::fs::create_dir_all(&patch).unwrap();
15748 }
15749 runner.state.save().unwrap();
15750 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15751 drop(pressure);
15752 runner.state = RunState::load(&runner.state.id).unwrap();
15753 assert_eq!(runner.state.status, RunStatus::Blocked);
15754 assert!(runner.state.interrupted_implementations.contains(&'A'));
15755 assert_eq!(runner.state.candidates[0].commits, 0);
15756 assert!(!runner.state.candidates[0].empty);
15757 assert!(runner.state.candidates[0].failed.is_none());
15758 if failed_checkpoint {
15759 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15760 std::fs::remove_dir(patch).unwrap();
15761 }
15762 runner.state.resource_block = None;
15763 runner.state.config.disk.min_free_bytes = 0;
15764 runner.roles.implementers[0].command = vec![
15765 "sh".into(),
15766 "-c".into(),
15767 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15768 ];
15769 runner.implement().await.unwrap();
15770 assert!(runner.state.interrupted_implementations.is_empty());
15771 assert!(runner.state.candidate_preservation.is_empty());
15772 assert_eq!(runner.state.candidates[0].commits, 1);
15773 assert!(!runner.state.candidates[0].empty);
15774 assert!(
15775 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15776 .unwrap()
15777 .contains("retried empty interrupted seat")
15778 );
15779 if !solo {
15780 assert_eq!(runner.state.candidates[1].commits, 1);
15781 }
15782 }
15783 }
15784
15785 #[tokio::test]
15786 #[cfg(unix)]
15787 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15788 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15789 let (_tmp, mut runner) = preservation_resume_fixture().await;
15790 runner.state.candidates.retain(|c| c.label == 'A');
15791 runner.state.candidates[0].empty = false;
15792 runner.state.candidates[0].summary.clear();
15793 runner.state.tally = None;
15794 runner.state.judge_skipped = false;
15795 runner.state.status = RunStatus::Implementing;
15796 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15799 let wt = runner.state.candidates[0].worktree.clone();
15800 std::fs::write(
15801 wt.join("interrupted-source"),
15802 "retain implementation work\n",
15803 )
15804 .unwrap();
15805 if committed {
15806 git::rescue_commit(&wt, "implementation checkpoint")
15807 .await
15808 .unwrap();
15809 }
15810 let lock = git::git(
15811 &wt,
15812 &[
15813 "rev-parse",
15814 "--path-format=absolute",
15815 "--git-path",
15816 "index.lock",
15817 ],
15818 )
15819 .await
15820 .unwrap();
15821 if block_rescue {
15822 std::fs::write(&lock, "fixture-owned lock").unwrap();
15823 }
15824 let mut implementer = spec("alpha");
15825 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15826 runner.roles.implementers = vec![implementer];
15827 runner.state.save().unwrap();
15828 let result = runner.implement().await;
15829 drop(pressure);
15830 assert_eq!(result.is_err(), block_rescue);
15831 assert_eq!(
15832 runner.state.resource_block.as_ref().unwrap().kind,
15833 "disk pressure"
15834 );
15835 assert!(
15836 runner
15837 .state
15838 .events
15839 .iter()
15840 .any(|e| e.message.contains("agent execution interrupted"))
15841 );
15842 assert!(
15843 runner
15844 .state
15845 .dir()
15846 .join("artifacts/candidate-A-recovery/staged.patch")
15847 .exists()
15848 );
15849 if block_rescue {
15850 let saved = RunState::load(&runner.state.id).unwrap();
15851 assert!(saved.candidate_preservation.contains_key(&'A'));
15852 assert!(!saved.candidates[0].empty);
15853 assert_eq!(
15854 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15855 "retain implementation work\n"
15856 );
15857 std::fs::remove_file(lock).unwrap();
15858 runner.state = saved;
15859 } else {
15860 assert_eq!(runner.state.status, RunStatus::Blocked);
15861 assert_eq!(runner.state.candidates[0].commits, 1);
15862 }
15863 runner.roles.implementers.clear();
15866 runner.state.config.disk.min_free_bytes = 0;
15867 runner.state.resource_block = None;
15868 runner.implement().await.unwrap();
15869 assert!(runner.state.candidate_preservation.is_empty());
15870 assert_eq!(runner.state.candidates[0].commits, 1);
15871 assert!(!runner.state.candidates[0].empty);
15872 assert!(git::is_clean(&wt).await.unwrap());
15873 assert!(
15874 crate::run::read_artifact(&runner.state, "cand-A.patch")
15875 .unwrap()
15876 .contains("retain implementation work")
15877 );
15878 }
15879 }
15880
15881 #[tokio::test]
15882 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15883 let (_tmp, mut runner) = preservation_resume_fixture().await;
15884 runner.implement().await.unwrap();
15886 assert!(runner.state.candidate_preservation.is_empty());
15887 assert!(
15888 !runner
15889 .state
15890 .dir()
15891 .join("artifacts/candidate-A-recovery")
15892 .exists()
15893 );
15894 runner.fold_losers().await.unwrap();
15895 assert!(!runner.state.candidates[0].worktree.exists());
15896 for schema in [14, crate::run::SCHEMA] {
15897 runner.state.schema = schema;
15898 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15899 runner.state.save().unwrap();
15900 runner.state = RunState::load(&runner.state.id).unwrap();
15901 runner.implement().await.unwrap();
15902 assert!(runner.state.candidate_preservation.is_empty());
15903 assert!(runner.state.candidates[0].folded);
15904 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15905 }
15906 }
15907
15908 #[tokio::test]
15909 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15910 let (_tmp, mut runner) = preservation_resume_fixture().await;
15911 runner.state.candidates.retain(|c| c.label == 'A');
15912 runner.state.tally = None;
15913 runner.state.judge_skipped = false;
15914 let candidate = &mut runner.state.candidates[0];
15915 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15916 candidate.failed = Some("deployment validation unavailable".into());
15917 std::fs::write(
15918 candidate.worktree.join("repair"),
15919 "future launcher policy\n",
15920 )
15921 .unwrap();
15922 runner
15923 .state
15924 .candidate_preservation
15925 .insert('A', "awaiting persistence".into());
15926 runner.persist_candidates().await.unwrap();
15927 let candidate = &runner.state.candidates[0];
15928 assert!(!candidate.empty);
15929 assert_eq!(candidate.commits, 1);
15930 assert_eq!(candidate.files, 1);
15931 assert!(candidate.failed.is_none());
15932 assert!(candidate.summary.contains("EPERM"));
15933 assert!(
15934 crate::run::read_artifact(&runner.state, "cand-A.patch")
15935 .unwrap()
15936 .contains("future launcher policy")
15937 );
15938 assert!(runner.state.candidate_preservation.is_empty());
15939 }
15940
15941 #[tokio::test]
15942 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15943 let (_tmp, mut runner) = preservation_resume_fixture().await;
15944 let wt = runner.state.candidates[0].worktree.clone();
15945 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15946 runner.state.schema = 14;
15947 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15948 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15949 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15950 for _ in 0..2 {
15951 assert!(runner.implement().await.is_err());
15952 assert_eq!(runner.state.status, RunStatus::Blocked);
15953 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15954 assert!(
15955 runner.state.candidates[0].empty,
15956 "old verdict must remain unusable"
15957 );
15958 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15959 assert!(runner.state.judge_skipped);
15960 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15961 assert!(
15962 crate::run::read_artifact(&runner.state, "cand-A.patch")
15963 .unwrap()
15964 .contains("owned edits")
15965 );
15966 assert!(wt.exists());
15967 assert_eq!(
15968 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15969 .await
15970 .unwrap(),
15971 1
15972 );
15973 runner.state = RunState::load(&runner.state.id).unwrap();
15974 }
15975 }
15976
15977 #[tokio::test]
15978 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15979 let (_tmp, mut runner) = preservation_resume_fixture().await;
15980 let wt = runner.state.candidates[0].worktree.clone();
15981 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15982 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15983 let index_lock = PathBuf::from(
15984 git::git(
15985 &wt,
15986 &[
15987 "rev-parse",
15988 "--path-format=absolute",
15989 "--git-path",
15990 "index.lock",
15991 ],
15992 )
15993 .await
15994 .unwrap(),
15995 );
15996 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15997 runner.state.schema = 14;
15998 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15999 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16000 assert!(runner.implement().await.is_err());
16001 assert!(runner.after_implement().is_err());
16002 assert!(runner.fold_losers().await.is_err());
16003 assert_eq!(
16004 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16005 "failed seat edits"
16006 );
16007 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16008 runner.state = RunState::load(&runner.state.id).unwrap();
16009 std::fs::remove_file(index_lock).unwrap();
16010 assert!(runner.implement().await.is_err());
16012 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16013 assert!(runner.state.candidates[0].empty);
16014 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16015 assert!(
16016 crate::run::read_artifact(&runner.state, "cand-A.patch")
16017 .unwrap()
16018 .contains("failed seat edits")
16019 );
16020 assert!(wt.exists());
16021 }
16022
16023 #[tokio::test]
16024 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16025 let (_tmp, mut runner) = preservation_resume_fixture().await;
16026 let repo = runner.state.repo.clone();
16027 let common = git::git(
16028 &repo,
16029 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16030 )
16031 .await
16032 .unwrap();
16033 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16034 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16035 assert!(runner.fold_losers().await.is_err());
16036 assert!(!runner.state.candidates[0].worktree.exists());
16037 assert!(!runner.state.candidates[0].folded);
16038 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16039 runner.state = RunState::load(&runner.state.id).unwrap();
16040 std::fs::remove_file(branch_lock).unwrap();
16041 let judge = runner.state.worktree_root().join("judge-1");
16042 git::worktree_add_detached(&repo, &judge, "main")
16043 .await
16044 .unwrap();
16045 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16046 runner.roles.judges = vec![spec("judge")];
16047 assert!(runner.fold_losers().await.is_err());
16048 runner.state = RunState::load(&runner.state.id).unwrap();
16049 assert!(
16050 runner.state.candidates[0].folded,
16051 "candidate progress saved before judge failure"
16052 );
16053 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16054 assert!(judge.join("uncommitted").exists());
16055 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16056 runner.fold_losers().await.unwrap();
16057 assert!(!judge.exists());
16058 runner.fold_losers().await.unwrap();
16059 }
16060
16061 #[tokio::test]
16062 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16063 let (_tmp, mut runner) = preservation_resume_fixture().await;
16064 runner.state.candidates.retain(|c| c.label == 'B');
16065 runner.state.tally = None;
16066 runner.state.judge_skipped = false;
16067 runner.state.prep_complete = Some(true);
16068 runner.state.config.graph.advise = false;
16069 runner.state.config.graph.selected_implementer = Some("alpha".into());
16070 let wt = runner.state.candidates[0].worktree.clone();
16071 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16072 runner
16073 .state
16074 .candidate_preservation
16075 .insert('B', "awaiting persistence".into());
16076 let index_lock = git::git(
16077 &wt,
16078 &[
16079 "rev-parse",
16080 "--path-format=absolute",
16081 "--git-path",
16082 "index.lock",
16083 ],
16084 )
16085 .await
16086 .unwrap();
16087 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16088 runner.state.save().unwrap();
16089 assert!(runner.execute_graph().await.is_err());
16090 assert_eq!(runner.state.status, RunStatus::Blocked);
16091 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16092 std::fs::remove_file(index_lock).unwrap();
16093 for _ in 0..2 {
16096 runner.state = RunState::load(&runner.state.id).unwrap();
16097 runner.pause.park();
16098 runner.execute_graph().await.unwrap();
16099 assert_eq!(runner.state.status, RunStatus::Implementing);
16100 assert!(runner.state.candidate_preservation.is_empty());
16101 assert_eq!(runner.state.candidates[0].commits, 2);
16102 assert!(runner.state.tally.is_none());
16103 }
16104 runner.implement().await.unwrap();
16105 assert_eq!(runner.state.status, RunStatus::Judging);
16106 runner.state.status = RunStatus::Blocked;
16108 runner.state.candidates[0].failed = Some("implementation failed".into());
16109 runner.state.save().unwrap();
16110 let error = runner.execute_graph().await.unwrap_err();
16111 assert!(
16112 error
16113 .to_string()
16114 .contains("selected implementation is held")
16115 );
16116 }
16117
16118 #[tokio::test]
16119 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16120 let (_tmp, mut runner) = preservation_resume_fixture().await;
16121 let repo = runner.state.repo.clone();
16122 let candidate = runner.state.candidates[0].clone();
16123 let registered = git::worktree_holding(&repo, &candidate.branch)
16126 .await
16127 .unwrap()
16128 .unwrap();
16129 assert_eq!(
16130 registered.canonicalize().unwrap(),
16131 candidate.worktree.canonicalize().unwrap()
16132 );
16133 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16134 assert_eq!(
16135 git::worktree_holding(&repo, &candidate.branch)
16136 .await
16137 .unwrap(),
16138 Some(registered)
16139 );
16140 assert!(
16141 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16142 "the stale registration initially keeps the branch checked out"
16143 );
16144 runner.fold_losers().await.unwrap();
16145 assert!(runner.state.candidates[0].folded);
16146 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16147 assert_eq!(
16148 git::worktree_holding(&repo, &candidate.branch)
16149 .await
16150 .unwrap(),
16151 None
16152 );
16153 runner.state = RunState::load(&runner.state.id).unwrap();
16154 runner.fold_losers().await.unwrap();
16155 }
16156
16157 #[tokio::test]
16158 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16159 let (_tmp, mut runner) = preservation_resume_fixture().await;
16160 let repo = runner.state.repo.clone();
16161 let candidate = runner.state.candidates[0].clone();
16162 let winner = runner.state.candidates[1].clone();
16163 let registered = git::worktree_holding(&repo, &candidate.branch)
16164 .await
16165 .unwrap()
16166 .unwrap();
16167 assert_eq!(
16168 registered.canonicalize().unwrap(),
16169 candidate.worktree.canonicalize().unwrap()
16170 );
16171 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16172 assert_eq!(
16173 git::worktree_holding(&repo, &candidate.branch)
16174 .await
16175 .unwrap(),
16176 Some(registered)
16177 );
16178 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16179
16180 for _ in 0..2 {
16181 fold_run(&mut runner.state, false, &crate::run::home())
16182 .await
16183 .unwrap();
16184 runner.state = RunState::load(&runner.state.id).unwrap();
16185 assert!(runner.state.candidates[0].folded);
16186 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16187 assert_eq!(
16188 git::worktree_holding(&repo, &candidate.branch)
16189 .await
16190 .unwrap(),
16191 None
16192 );
16193 assert!(!runner.state.candidates[1].folded);
16194 assert!(winner.worktree.exists());
16195 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16196 }
16197 }
16198
16199 #[tokio::test]
16200 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16201 ask_test_home();
16202 let tmp = tempfile::tempdir().unwrap();
16203 let repo = tmp.path().join("repo");
16204 std::fs::create_dir(&repo).unwrap();
16205 init_repo(&repo);
16206 let mut runner = runner_at(RunStatus::Implementing);
16207 runner.state.repo = repo.clone();
16208 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16209 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16210 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16211 let branch = format!("candidate-{label}");
16212 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16213 git::worktree_add_branch(&repo, &wt, &branch, "main")
16214 .await
16215 .unwrap();
16216 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16217 git::git(&wt, &["add", "README.md"]).await.unwrap();
16218 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16219 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16220 runner.state.candidates.push(Candidate {
16221 index,
16222 label,
16223 agent: "alpha".to_owned(),
16224 branch,
16225 worktree: wt,
16226 summary: "completed implementation".to_owned(),
16227 stat: String::new(),
16228 files: 0,
16229 commits: 0,
16230 empty: false,
16231 failed: None,
16232 verified_noop: None,
16233 duration_ms: 42,
16234 folded: false,
16235 });
16236 runner
16237 .state
16238 .candidate_preservation
16239 .insert(label, "awaiting persistence".to_owned());
16240 }
16241 let wt = runner.state.candidates[0].worktree.clone();
16242 let index_lock = git::git(
16243 &wt,
16244 &[
16245 "rev-parse",
16246 "--path-format=absolute",
16247 "--git-path",
16248 "index.lock",
16249 ],
16250 )
16251 .await
16252 .unwrap();
16253 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16254 runner.state.save().unwrap();
16255 assert!(runner.persist_candidates().await.is_err());
16256 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16257 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16258 assert_eq!(
16259 runner.state.candidates[1].commits, 1,
16260 "other results are still collected"
16261 );
16262 assert!(
16263 !runner.state.candidates[0].empty,
16264 "failed rescue is not no change"
16265 );
16266 assert_eq!(
16267 runner.state.candidates[0].summary,
16268 "completed implementation"
16269 );
16270 assert!(runner.after_implement().is_err());
16271 let home = tmp.path().join("fold-home");
16272 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16273 assert!(wt.join("new-file").exists());
16274 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16275 assert!(recovery.join("staged.patch").exists());
16276 assert!(recovery.join("unstaged.patch").exists());
16277 assert_eq!(
16278 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16279 "untracked edits\n"
16280 );
16281 runner.state.begin_pass();
16282 runner.state.end_pass_parked(ParkCause::Upgrade {
16283 from: "1.0".into(),
16284 to: Some("1.1".into()),
16285 });
16286 runner.state.save().unwrap();
16287 runner.state = RunState::load(&runner.state.id).unwrap();
16288 assert!(matches!(
16289 runner.state.passes.last().unwrap().end,
16290 Some(crate::run::PassEnd::Parked {
16291 cause: ParkCause::Upgrade { .. }
16292 })
16293 ));
16294 std::fs::remove_file(index_lock).unwrap();
16295 runner.implement().await.unwrap();
16297 assert!(runner.state.candidate_preservation.is_empty());
16298 assert!(
16299 runner
16300 .state
16301 .candidates
16302 .iter()
16303 .all(|c| c.commits == 1 && !c.empty)
16304 );
16305 assert!(git::is_clean(&wt).await.unwrap());
16306 assert!(
16307 crate::run::read_artifact(&runner.state, "cand-A.patch")
16308 .unwrap()
16309 .contains("later tracked edits")
16310 );
16311 }
16312
16313 #[tokio::test]
16314 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16315 ask_test_home();
16316 let tmp = tempfile::tempdir().unwrap();
16317 let repo = tmp.path().join("repo");
16318 std::fs::create_dir(&repo).unwrap();
16319 init_repo(&repo);
16320 let wt = tmp.path().join("candidate");
16321 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16322 .await
16323 .unwrap();
16324 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16325 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16326 assert!(wt.join("uncommitted").exists());
16327 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16328 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16329 let mut runner = runner_at(RunStatus::Implementing);
16330 runner.state.repo = repo.clone();
16331 runner.state.base_commit = "missing-base".to_owned();
16332 runner.state.candidates.push(Candidate {
16333 index: 0,
16334 label: 'A',
16335 agent: "alpha".into(),
16336 branch: "candidate".into(),
16337 worktree: wt.clone(),
16338 summary: "completed".into(),
16339 stat: String::new(),
16340 files: 0,
16341 commits: 0,
16342 empty: false,
16343 failed: None,
16344 verified_noop: None,
16345 duration_ms: 1,
16346 folded: false,
16347 });
16348 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16349 assert!(
16350 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16351 .await
16352 .is_err()
16353 );
16354 assert!(!runner.state.candidates[0].folded);
16355 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16356 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16357 runner
16358 .state
16359 .candidate_preservation
16360 .insert('A', "pending".into());
16361 assert!(runner.persist_candidates().await.is_err());
16362 assert!(!runner.state.candidates[0].empty);
16363 assert!(!runner.state.candidates[0].folded);
16364 assert!(wt.exists());
16365 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16366 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16367 std::fs::create_dir_all(&patch_path).unwrap();
16368 assert!(
16369 runner.persist_candidates().await.is_err(),
16370 "artifact failure stays pending"
16371 );
16372 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16373 assert!(!runner.state.candidates[0].empty);
16374 std::fs::remove_dir(patch_path).unwrap();
16375 runner.persist_candidates().await.unwrap();
16376 assert!(
16377 runner.state.candidates[0].empty,
16378 "only successful inspection confirms no change"
16379 );
16380 }
16381
16382 fn init_repo(dir: &Path) {
16383 let run = |args: &[&str]| {
16384 let out = std::process::Command::new("git")
16385 .args(args)
16386 .current_dir(dir)
16387 .quiet()
16388 .output()
16389 .expect("spawn git");
16390 assert!(
16391 out.status.success(),
16392 "git {args:?} failed: {}",
16393 String::from_utf8_lossy(&out.stderr)
16394 );
16395 };
16396 run(&["init", "-b", "main"]);
16397 run(&["config", "user.name", "magi test"]);
16398 run(&["config", "user.email", "magi@example.com"]);
16399 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16400 run(&["add", "-A"]);
16401 run(&["commit", "-m", "init"]);
16402 }
16403
16404 fn ask_test_home() {
16412 crate::run::pin_test_home();
16413 }
16414
16415 #[test]
16416 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16417 ask_test_home();
16418 let mut runner = runner_at(RunStatus::Prep);
16419 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16420 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16423 let error = guard
16424 .finish(
16425 &mut runner.state,
16426 Err(
16427 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16428 .into(),
16429 ),
16430 )
16431 .unwrap_err();
16432 assert_eq!(
16433 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16434 std::io::ErrorKind::PermissionDenied
16435 );
16436 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16437 let message = format!("{error:#}");
16438 assert!(message.contains("original refusal"));
16439 assert!(!message.contains("could not save terminal"));
16440 assert_eq!(
16441 error.chain().count(),
16442 2,
16443 "only startup attribution wraps the original error"
16444 );
16445 let saved = RunState::load(&runner.state.id).unwrap();
16446 assert_eq!(saved.status, RunStatus::Blocked);
16447 assert!(!saved.startup.unwrap().complete);
16448 }
16449
16450 #[test]
16451 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16452 ask_test_home();
16453 let mut runner = runner_at(RunStatus::Prep);
16454 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16455 let id = runner.state.id.clone();
16456 drop(guard);
16457 let saved = RunState::load(&id).unwrap();
16458 assert_eq!(saved.status, RunStatus::Blocked);
16459 assert!(!saved.startup.as_ref().unwrap().complete);
16460 assert!(
16461 saved
16462 .startup
16463 .unwrap()
16464 .failure
16465 .unwrap()
16466 .contains("interrupted")
16467 );
16468 assert!(Runner::resume(&id).is_err());
16469 }
16470
16471 #[test]
16472 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16473 ask_test_home();
16474 let mut runner = runner_at(RunStatus::Prep);
16475 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16476 let error = guard
16477 .finish(
16478 &mut runner.state,
16479 Err(
16480 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16481 .into(),
16482 ),
16483 )
16484 .unwrap_err();
16485 assert_eq!(
16486 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16487 std::io::ErrorKind::PermissionDenied
16488 );
16489 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16490 let mut task = crate::queue::Task::new(
16491 "review".into(),
16492 "criteria".into(),
16493 runner.state.repo.clone(),
16494 crate::queue::Source::Human,
16495 );
16496 task.record_startup_failure(failure.clone());
16497 task.record_startup_failure(failure.clone());
16498 assert_eq!(task.startup_failures.len(), 1);
16499 assert!(task.runs.is_empty());
16500 assert_eq!(task.attempts, 0);
16501 }
16502
16503 #[cfg(unix)]
16504 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16505 ask_test_home();
16506 let tmp = tempfile::tempdir().unwrap();
16507 let repo = tmp.path().join("repo");
16508 std::fs::create_dir(&repo).unwrap();
16509 init_repo(&repo);
16510 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16511 let origin = tmp.path().join("origin.git");
16512 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16513 .await
16514 .unwrap();
16515 git::git(
16516 &repo,
16517 &["remote", "add", "origin", origin.to_str().unwrap()],
16518 )
16519 .await
16520 .unwrap();
16521 git::git(&repo, &["checkout", "-b", "fix/retained"])
16522 .await
16523 .unwrap();
16524 std::fs::write(repo.join("retained"), "retained change").unwrap();
16525 git::git(&repo, &["add", "."]).await.unwrap();
16526 git::git(&repo, &["commit", "-m", "retained implementation"])
16527 .await
16528 .unwrap();
16529 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16530 git::git(&repo, &["checkout", "main"]).await.unwrap();
16531 std::fs::write(repo.join("base-change"), "new base").unwrap();
16532 git::git(&repo, &["add", "."]).await.unwrap();
16533 git::git(&repo, &["commit", "-m", "base moved"])
16534 .await
16535 .unwrap();
16536 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16537 git::git(&repo, &["checkout", "fix/retained"])
16538 .await
16539 .unwrap();
16540 let mut runner = runner_at(RunStatus::Voting);
16541 runner.state.repo = repo.clone();
16542 runner.state.base_commit = base;
16543 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16544 runner.state.config.disk.min_free_bytes = 0;
16545 runner.state.config.graph.review_rounds = 1;
16546 runner.state.config.verify.e2e = vec!["true".into()];
16547 runner.state.config.verify.gate = vec!["true".into()];
16548 runner.state.candidates = vec![Candidate {
16549 index: 0,
16550 label: 'A',
16551 agent: "fixture".into(),
16552 branch: "fix/retained".into(),
16553 worktree: repo.clone(),
16554 summary: String::new(),
16555 stat: String::new(),
16556 files: 1,
16557 commits: 1,
16558 empty: false,
16559 failed: None,
16560 verified_noop: None,
16561 duration_ms: 0,
16562 folded: false,
16563 }];
16564 runner.state.tally = Some(Tally {
16565 first_choice: BTreeMap::new(),
16566 borda: BTreeMap::new(),
16567 winner: 'A',
16568 rankings: 0,
16569 unanimous_initial: true,
16570 deliberated: false,
16571 changed_votes: 0,
16572 unanimous_final: true,
16573 tie_break: None,
16574 judges: 0,
16575 present: 0,
16576 quorum: 0,
16577 met_quorum: true,
16578 uncontested: Some("review-only".into()),
16579 });
16580 let mut task = crate::queue::Task::new(
16581 "fixture".into(),
16582 "fixture".into(),
16583 repo,
16584 crate::queue::Source::Human,
16585 );
16586 task.link_run(&runner.state.id);
16587 crate::queue::Queue::open().put(&mut task).unwrap();
16588 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16589 runner.state.base_sync = Some(BaseSync {
16590 tip: String::new(),
16591 behind: 1,
16592 attempts: BASE_SYNC_ROUNDS,
16593 conflict: None,
16594 already_in: None,
16595 });
16596 runner.sync_to_base().await.unwrap();
16597 assert_eq!(runner.state.status, RunStatus::Blocked);
16598 (tmp, runner, head, task.id)
16599 }
16600
16601 #[cfg(unix)]
16602 #[tokio::test]
16603 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16604 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16605 if std::env::var_os(CHILD).is_none() {
16606 let home = tempfile::tempdir().unwrap();
16607 let output = std::process::Command::new(std::env::current_exe().unwrap())
16608 .args([
16609 "--exact",
16610 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16611 "--nocapture",
16612 ])
16613 .env(CHILD, "1")
16614 .env("MAGI_HOME", home.path())
16615 .env("MAGI_WORKER_HOME", home.path())
16616 .env("MAGI_WORKER_ID", "abcdef")
16617 .output()
16618 .unwrap();
16619 assert!(
16620 output.status.success()
16621 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16622 "{}\n{}",
16623 String::from_utf8_lossy(&output.stdout),
16624 String::from_utf8_lossy(&output.stderr)
16625 );
16626 return;
16627 }
16628 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16629 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16630 let home = crate::run::home();
16631 let queue = crate::queue::Queue::open();
16632 let mut owner = queue.get(&task).unwrap();
16633 owner.release_operator("test-release", None);
16634 owner.start(runner.state.id.clone());
16635 queue.put(&mut owner).unwrap();
16636 runner.state.driver_exited = true;
16637 runner.state.task_instruction = Some(owner.instruction.clone());
16638 runner.state.save().unwrap();
16639 let _claim = queue.claim(&task).unwrap();
16640 let reserve = |id: &str| {
16641 let directory = home.join("workers/jobs").join(id);
16642 std::fs::create_dir_all(&directory).unwrap();
16643 let job = crate::worker::Job {
16644 id: id.into(),
16645 sequence: 1,
16646 kind: "task".into(),
16647 key: format!("task:{task}"),
16648 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16649 generation: "fixture-generation".into(),
16650 version: None,
16651 status: crate::worker::Status::Running,
16652 result: None,
16653 pid: Some(std::process::id()),
16654 created_at: 0,
16655 finished_at: None,
16656 };
16657 std::fs::write(
16658 directory.join("job.json"),
16659 serde_json::to_vec(&job).unwrap(),
16660 )
16661 .unwrap();
16662 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16663 };
16664 let _own = reserve("abcdef");
16665 let other = reserve("fedcba");
16666 assert!(
16667 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16668 .await
16669 .is_err()
16670 );
16671 assert!(
16672 RunState::load(&runner.state.id)
16673 .unwrap()
16674 .base_sync_recovery
16675 .authorizations
16676 .is_empty()
16677 );
16678 drop(other);
16679 assert!(crate::daemon::is_working_on(
16680 &home,
16681 &runner.state.id,
16682 Timestamp::now()
16683 ));
16684 assert!(
16685 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16686 );
16687 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16688 .await
16689 .unwrap();
16690 assert!(
16691 RunState::load(&runner.state.id)
16692 .unwrap()
16693 .base_recovery_pending()
16694 );
16695 }
16696
16697 #[cfg(unix)]
16698 #[tokio::test]
16699 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16700 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16701 let home = crate::run::home();
16702 runner.state.driver_pid = Some(std::process::id());
16703 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16704 runner.state.driver_exited = false;
16705 runner.state.begin_pass();
16706 runner.state.save().unwrap();
16707 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16708 driver.record(&runner.state).unwrap();
16709 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16710 drop(driver);
16712 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16713 assert_eq!(
16714 runner.state.liveness_under(&home, false),
16715 crate::run::Liveness::Dead
16716 );
16717 let queue = crate::queue::Queue::open();
16718 let _claim = queue.claim(&task).unwrap();
16719 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16720 .await
16721 .unwrap();
16722 let disk = RunState::load(&runner.state.id).unwrap();
16723 assert!(!disk.driver_exited);
16724 assert!(disk.base_recovery_pending());
16725 }
16726
16727 #[cfg(unix)]
16728 #[tokio::test]
16729 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16730 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16731 if std::env::var_os(CHILD).is_none() {
16732 use std::os::unix::fs::PermissionsExt;
16733 let tmp = tempfile::tempdir().unwrap();
16734 let gh = tmp.path().join("gh");
16735 std::fs::write(&gh, r#"#!/bin/sh
16736case "$1 $2" in
16737 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16738 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16739 *) exit 2 ;;
16740esac
16741"#).unwrap();
16742 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16743 let mut paths = vec![tmp.path().to_path_buf()];
16744 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16745 let output = std::process::Command::new(std::env::current_exe().unwrap())
16746 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16747 .env(CHILD, "1")
16748 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16749 .env("PATH", std::env::join_paths(paths).unwrap())
16750 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16751 .output().unwrap();
16752 assert!(
16753 output.status.success()
16754 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16755 "{}\n{}",
16756 String::from_utf8_lossy(&output.stdout),
16757 String::from_utf8_lossy(&output.stderr)
16758 );
16759 return;
16760 }
16761 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16762 let repo = runner.state.repo.clone();
16763 git::git(&repo, &["checkout", "main"]).await.unwrap();
16764 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16765 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16766 git::git(&repo, &["checkout", "fix/retained"])
16767 .await
16768 .unwrap();
16769 runner.state.pr = Some(crate::run::PrRecord {
16770 url: "https://example.test/project/pull/1".into(),
16771 number: 1,
16772 state: "open".into(),
16773 checks: "unknown".into(),
16774 round: 0,
16775 rounds: 1,
16776 red_at_merge: Vec::new(),
16777 });
16778 runner.sync_to_base().await.unwrap();
16779 assert_eq!(runner.state.status, RunStatus::Blocked);
16780 assert_eq!(
16781 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16782 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16783 );
16784 assert!(runner.state.base_sync_closure_pending());
16785 assert!(crate::handover::can_resume(&runner.state));
16786 let queue = crate::queue::Queue::open();
16787 assert!(!crate::base_sync_recovery::held_task(
16788 &queue,
16789 &queue.get(&task).unwrap()
16790 ));
16791 assert!(runner.state.authorize_base_sync(&task).is_err());
16792 runner.state.base_sync_recovery.stop = None;
16794 runner.state.status = RunStatus::Voting;
16795 runner.state.gate_ran = true;
16796 runner.state.config.graph.review_rounds = 0;
16797 runner.state.save().unwrap();
16798 assert!(crate::handover::can_resume(&runner.state));
16799 runner.state = RunState::load(&runner.state.id).unwrap();
16800 runner.execute_graph().await.unwrap();
16801 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16802 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16803 assert_eq!(
16804 runner.state.base_sync.as_ref().unwrap().attempts,
16805 BASE_SYNC_ROUNDS
16806 );
16807 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16808 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16809 assert!(runner.state.reviews.is_empty());
16810 assert!(runner.state.gate.is_empty());
16811 }
16812
16813 #[cfg(unix)]
16814 #[tokio::test]
16815 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16816 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16817 runner.state.driver_exited = true;
16818 runner.state.save().unwrap();
16819 let home = crate::run::home();
16820 let queue = crate::queue::Queue::open();
16821 let _claim = queue.claim(&task).unwrap();
16822 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16823 assert!(
16824 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16825 .await
16826 .is_err()
16827 );
16828 drop(lease);
16829 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16830 assert!(
16831 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16832 .await
16833 .is_err()
16834 );
16835 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16836 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16837 .await
16838 .unwrap();
16839 let disk = RunState::load(&runner.state.id).unwrap();
16840 assert!(disk.base_recovery_pending());
16841 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16842 assert!(
16843 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16844 .await
16845 .is_err()
16846 );
16847 assert_eq!(
16848 RunState::load(&runner.state.id)
16849 .unwrap()
16850 .base_sync_recovery
16851 .authorizations
16852 .len(),
16853 1
16854 );
16855 let mut owner = queue.get(&task).unwrap();
16856 owner.runs.push("20260101-000000-ffff".into());
16857 queue.put(&mut owner).unwrap();
16858 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16859 .await
16860 .unwrap_err();
16861 assert!(error.to_string().contains("run task ownership changed"));
16862 }
16863
16864 #[cfg(unix)]
16865 #[tokio::test]
16866 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16867 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16868 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16869 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16870 legacy["status"] = serde_json::json!("voting");
16871 runner.state = serde_json::from_value(legacy).unwrap();
16872 runner.state.save().unwrap();
16873 for _ in 0..3 {
16874 runner.execute_graph().await.unwrap();
16875 assert_eq!(runner.state.status, RunStatus::Blocked);
16876 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16877 assert!(!crate::handover::can_resume(&runner.state));
16878 assert_eq!(
16879 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16880 head
16881 );
16882 assert!(runner.state.reviews.is_empty());
16883 assert!(!runner.state.gate_ran);
16884 }
16885 let notice = crate::notices::run_ended(&runner.state).unwrap();
16886 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16887 runner.state.begin_pass();
16888 runner.state.status = RunStatus::Voting;
16889 runner.deliberate().await.unwrap();
16890 runner.vote().await.unwrap();
16891 runner.finish_after_tally().await.unwrap();
16892 assert_eq!(runner.state.status, RunStatus::Blocked);
16893 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16894 assert_eq!(
16895 crate::notices::run_ended(&runner.state).unwrap().since,
16896 notice.since
16897 );
16898 }
16899
16900 #[cfg(unix)]
16901 #[tokio::test]
16902 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16903 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16904 for detail in [
16905 "fix/retained conflicts with origin/main and did not rebase: conflict",
16906 "rebased fix/retained locally but origin refused the push",
16907 "unknown failure",
16908 ] {
16909 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16910 runner.state.record_base_sync_stop(&head);
16911 assert!(runner.state.authorize_base_sync(&task).is_err());
16912 runner.state.status = RunStatus::Voting;
16913 runner.execute_graph().await.unwrap();
16914 assert_eq!(runner.state.status, RunStatus::Blocked);
16915 assert!(!crate::handover::can_resume(&runner.state));
16916 assert_eq!(
16917 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16918 head
16919 );
16920 }
16921 }
16922
16923 #[cfg(unix)]
16924 #[tokio::test]
16925 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16926 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16927 runner.state.authorize_base_sync(&task).unwrap();
16928 assert!(runner.state.consume_base_recovery(&head));
16929 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16930 runner.state.reserve_base_operation(&head, &tip);
16931 runner.state.save().unwrap();
16932 runner.state = RunState::load(&runner.state.id).unwrap();
16933 assert!(!runner.state.consume_base_recovery(&head));
16934 runner.sync_to_base().await.unwrap();
16935 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16936 assert_eq!(runner.state.base_episode_used(), 1);
16937 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16938 assert!(
16939 runner
16940 .state
16941 .base_sync_recovery
16942 .operations
16943 .last()
16944 .unwrap()
16945 .finished
16946 );
16947 assert!(!runner.state.base_sync_blocked());
16948 runner.sync_to_base().await.unwrap();
16949 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16950 }
16951
16952 #[cfg(unix)]
16953 #[tokio::test]
16954 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16955 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16956 let original = runner
16958 .state
16959 .base_sync
16960 .as_ref()
16961 .unwrap()
16962 .conflict
16963 .clone()
16964 .unwrap();
16965 runner.state.base_sync.as_mut().unwrap().conflict = Some(
16966 original
16967 .replace("synchronization(s)", "rebase(s)")
16968 .replace("synchronizing", "rebasing"),
16969 );
16970 runner.state.base_sync_recovery = Default::default();
16971 runner.state.record_base_sync_stop(&head);
16972 assert_eq!(
16973 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16974 crate::base_sync_recovery::StopKind::Exhausted
16975 );
16976 runner.state.authorize_base_sync(&task).unwrap();
16977 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16978 let auth = legacy["base_sync_recovery"]["authorizations"][0]
16979 .as_object_mut()
16980 .unwrap();
16981 auth.remove("interval");
16982 auth.remove("scope");
16983 runner.state = serde_json::from_value(legacy).unwrap();
16984 assert!(runner.state.consume_base_recovery(&head));
16985 assert_eq!(runner.state.base_episode_used(), 3);
16986 runner.sync_to_base().await.unwrap();
16987 assert_eq!(runner.state.base_episode_used(), 4);
16988 runner.state.resource_block = Some(crate::run::ResourceBlock {
16989 node: "gate".into(),
16990 kind: "cache".into(),
16991 detail: "legacy wait".into(),
16992 at: Timestamp::now(),
16993 cause: crate::run::ResourceBlockCause::Unrecorded,
16994 });
16995 runner.state.save().unwrap();
16996 assert!(
16997 runner
16998 .state
16999 .base_sync_recovery
17000 .episodes
17001 .last()
17002 .unwrap()
17003 .boundary
17004 .is_none()
17005 );
17006 runner.state.resource_block = None;
17007 advance_base_fixture(&runner).await;
17008 runner.sync_to_base().await.unwrap();
17009 assert!(runner.state.base_sync_blocked());
17010 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17011 }
17012
17013 #[cfg(unix)]
17014 async fn advance_base_fixture(runner: &Runner) {
17015 let repo = &runner.state.repo;
17016 for args in [
17017 vec!["checkout", "main"],
17018 vec!["commit", "--allow-empty", "-m", "base advanced"],
17019 vec!["push", "origin", "main"],
17020 vec!["checkout", "fix/retained"],
17021 ] {
17022 git::git(repo, &args).await.unwrap();
17023 }
17024 }
17025
17026 #[cfg(unix)]
17027 fn fresh_base_interval_fixture(runner: &mut Runner) {
17028 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17029 runner.state.base_sync.as_mut().unwrap().conflict = None;
17030 runner.state.base_sync_recovery = Default::default();
17031 runner.state.status = RunStatus::Voting;
17032 }
17033
17034 #[cfg(unix)]
17035 #[tokio::test]
17036 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17037 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17038 fresh_base_interval_fixture(&mut runner);
17039 runner.state.begin_pass();
17040 runner.sync_to_base().await.unwrap();
17041 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17042 runner.state.resource_block = Some(crate::run::ResourceBlock {
17043 node: "gate".into(),
17044 kind: "build cache".into(),
17045 detail: "busy".into(),
17046 at: Timestamp::now(),
17047 cause: crate::run::ResourceBlockCause::Timeout {
17048 waited_secs: 1,
17049 limit_secs: 1,
17050 },
17051 });
17052 runner.state.save().unwrap();
17053 let queue = crate::queue::Queue::open();
17054 let mut owner = queue.get(&task).unwrap();
17055 runner.state.task_instruction = Some(owner.instruction.clone());
17056 owner.release_operator("fixture-owner", None);
17057 queue.put(&mut owner).unwrap();
17058 let approval = owner.resumption.clone().unwrap();
17059 runner.state.resource_block = None;
17060 runner.state.save().unwrap();
17061 runner.state = RunState::load(&runner.state.id).unwrap();
17062 advance_base_fixture(&runner).await;
17063 runner.sync_to_base().await.unwrap();
17064 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17065 assert_eq!(runner.state.base_episode_used(), 1);
17066 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17067 assert!(
17068 runner.state.base_sync_recovery.episodes[0]
17069 .boundary
17070 .as_ref()
17071 .unwrap()
17072 .consumed
17073 );
17074 assert!(
17075 !runner
17076 .state
17077 .renew_base_episode(&task, &approval, &head, "base")
17078 );
17079 for _ in 0..3 {
17080 owner.release_operator("fixture-owner", None);
17081 queue.put(&mut owner).unwrap();
17082 runner.state = RunState::load(&runner.state.id).unwrap();
17083 runner.state.begin_pass();
17084 runner.sync_to_base().await.unwrap();
17085 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17086 assert_eq!(runner.state.base_episode_used(), 1);
17087 }
17088 runner.state.resource_block = Some(crate::run::ResourceBlock {
17089 node: "gate".into(),
17090 kind: "disk pressure".into(),
17091 detail: "free space required".into(),
17092 at: Timestamp::now(),
17093 cause: crate::run::ResourceBlockCause::Other,
17094 });
17095 runner.state.end_pass_parked(ParkCause::Interrupt {
17096 reason: "disk pressure".into(),
17097 });
17098 runner.state.save().unwrap();
17099 owner.release_operator("fixture-owner-after-disk", None);
17100 queue.put(&mut owner).unwrap();
17101 runner.state.resource_block = None;
17102 advance_base_fixture(&runner).await;
17103 runner.sync_to_base().await.unwrap();
17104 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17105 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17106 assert_eq!(runner.state.base_episode_used(), 1);
17107 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17108 }
17109
17110 #[cfg(unix)]
17111 #[tokio::test]
17112 async fn base_recovery_continuous_advancement_stops_at_four() {
17113 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17114 fresh_base_interval_fixture(&mut runner);
17115 for spent in 1..=BASE_SYNC_ROUNDS {
17116 runner.sync_to_base().await.unwrap();
17117 assert!(!runner.state.base_sync_blocked());
17118 assert_eq!(runner.state.base_episode_used(), spent);
17119 advance_base_fixture(&runner).await;
17120 }
17121 runner.sync_to_base().await.unwrap();
17122 assert_eq!(
17123 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17124 crate::base_sync_recovery::StopKind::Exhausted
17125 );
17126 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17127 assert!(stop.unverified_sync);
17128 let action = runner.state.base_sync_next_action();
17129 assert!(action.contains("never reached its gate"));
17130 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17131 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17132 runner.state = RunState::load(&runner.state.id).unwrap();
17133 runner.sync_to_base().await.unwrap();
17134 assert_eq!(runner.state.base_episode_used(), 4);
17135 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17136 }
17137
17138 #[cfg(unix)]
17139 #[tokio::test]
17140 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17141 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17142 runner.state.ensure_base_episode();
17143 runner.state.record_base_progress(&head);
17145 runner.state.base_sync_recovery.stop = None;
17146 runner.state.record_base_sync_stop(&head);
17147 assert!(
17148 !runner
17149 .state
17150 .base_sync_recovery
17151 .stop
17152 .as_ref()
17153 .unwrap()
17154 .unverified_sync
17155 );
17156
17157 runner.state.finish_base_operation(Some(&head));
17158 runner.state.gate_ran = true;
17160 runner.state.gate_head = Some("older-head".into());
17161 runner.state.base_sync_recovery.stop = None;
17162 runner.state.record_base_sync_stop(&head);
17163 runner.state.save().unwrap();
17164 runner.state = RunState::load(&runner.state.id).unwrap();
17165 assert!(
17166 runner
17167 .state
17168 .base_sync_recovery
17169 .stop
17170 .as_ref()
17171 .unwrap()
17172 .unverified_sync
17173 );
17174
17175 runner.state.gate_head = Some(head.clone());
17176 runner.state.base_sync_recovery.stop = None;
17177 runner.state.record_base_sync_stop(&head);
17178 assert!(
17179 !runner
17180 .state
17181 .base_sync_recovery
17182 .stop
17183 .as_ref()
17184 .unwrap()
17185 .unverified_sync
17186 );
17187 assert!(
17188 !runner
17189 .state
17190 .base_sync_next_action()
17191 .contains("never reached its gate")
17192 );
17193
17194 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17196 legacy["base_sync_recovery"]["stop"]
17197 .as_object_mut()
17198 .unwrap()
17199 .remove("unverified_sync");
17200 legacy["base_sync_recovery"]["episodes"][0]
17201 .as_object_mut()
17202 .unwrap()
17203 .remove("last_sync_head");
17204 runner.state = serde_json::from_value(legacy).unwrap();
17205 assert!(
17206 !runner
17207 .state
17208 .base_sync_recovery
17209 .stop
17210 .as_ref()
17211 .unwrap()
17212 .unverified_sync
17213 );
17214 assert!(
17215 runner.state.base_sync_recovery.episodes[0]
17216 .last_sync_head
17217 .is_none()
17218 );
17219 }
17220
17221 #[cfg(unix)]
17222 #[tokio::test]
17223 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17224 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17225 runner.state.authorize_base_sync(&task).unwrap();
17226 assert!(
17227 runner
17228 .state
17229 .base_sync_next_action()
17230 .contains("Approval recorded")
17231 );
17232 assert!(
17233 runner
17234 .state
17235 .base_sync_next_action()
17236 .contains("4/4 in episode 0, 4 lifetime")
17237 );
17238 assert!(runner.state.consume_base_recovery(&head));
17239 assert!(
17240 runner
17241 .state
17242 .base_sync_next_action()
17243 .contains("0/4 in episode 1, 4 lifetime")
17244 );
17245 }
17246
17247 #[cfg(unix)]
17248 #[tokio::test]
17249 async fn base_recovery_changed_head_invalidates_without_spending() {
17250 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17251 runner.state.authorize_base_sync(&task).unwrap();
17252 git::git(
17253 &runner.state.repo,
17254 &["commit", "--allow-empty", "-m", "external head change"],
17255 )
17256 .await
17257 .unwrap();
17258 runner.sync_to_base().await.unwrap();
17259 let a = runner
17260 .state
17261 .base_sync_recovery
17262 .authorizations
17263 .last()
17264 .unwrap();
17265 assert!(a.consumed);
17266 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17267 runner.state = RunState::load(&runner.state.id).unwrap();
17268 assert!(
17269 runner
17270 .state
17271 .base_sync_next_action()
17272 .contains("Approval invalidated")
17273 );
17274 assert!(!runner.state.base_recovery_pending());
17275 assert!(runner.state.base_sync_blocked());
17276 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17277 }
17278
17279 #[cfg(unix)]
17280 #[tokio::test]
17281 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17282 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17283 runner.state.driver_exited = true;
17284 runner.state.save().unwrap();
17285 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17286 .await
17287 .unwrap();
17288 runner.state = RunState::load(&runner.state.id).unwrap();
17289 advance_base_fixture(&runner).await;
17290 runner.sync_to_base().await.unwrap();
17291 let a = runner
17292 .state
17293 .base_sync_recovery
17294 .authorizations
17295 .last()
17296 .unwrap();
17297 assert!(
17298 a.consumed
17299 && a.invalidated
17300 .as_deref()
17301 .unwrap()
17302 .contains("fetched base changed")
17303 );
17304 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17305 assert_eq!(
17306 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17307 head
17308 );
17309 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17310 .await
17311 .unwrap();
17312 runner.state = RunState::load(&runner.state.id).unwrap();
17313 runner.sync_to_base().await.unwrap();
17314 assert!(!runner.state.base_sync_blocked());
17315 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17316 }
17317
17318 #[cfg(unix)]
17319 #[tokio::test]
17320 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17321 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17322 runner.state.authorize_base_sync(&task).unwrap();
17323 assert!(runner.state.consume_base_recovery(&head));
17324 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17325 runner.state.reserve_base_operation(&head, &tip);
17326 runner.state.save().unwrap();
17327 let scratch = runner.state.dir().join("base-sync");
17328 assert_eq!(
17329 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17330 .await
17331 .unwrap(),
17332 crate::rebase::Rebased::Applied
17333 );
17334 runner.state = RunState::load(&runner.state.id).unwrap();
17335 runner.sync_to_base().await.unwrap();
17336 assert!(
17337 runner
17338 .state
17339 .base_sync_recovery
17340 .operations
17341 .last()
17342 .unwrap()
17343 .finished
17344 );
17345 assert_eq!(runner.state.base_episode_used(), 1);
17346 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17347 assert!(!runner.state.base_sync_blocked());
17348 }
17349
17350 #[cfg(unix)]
17351 #[tokio::test]
17352 async fn base_recovery_crash_preserves_untracked_files() {
17353 for ignored in [false, true] {
17354 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17355 runner.state.authorize_base_sync(&task).unwrap();
17356 assert!(runner.state.consume_base_recovery(&head));
17357 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17358 runner.state.reserve_base_operation(&head, &tip);
17359 runner.state.save().unwrap();
17360 let scratch = runner.state.dir().join("base-sync");
17361 assert_eq!(
17362 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17363 .await
17364 .unwrap(),
17365 crate::rebase::Rebased::Applied
17366 );
17367 let repo = &runner.state.repo;
17368 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17369 assert_ne!(result, head);
17370 if ignored {
17371 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17372 .await
17373 .unwrap();
17374 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17375 }
17376 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17377 assert!(
17378 git::git_raw(repo, &["diff", "--quiet", &head])
17379 .await
17380 .unwrap()
17381 .ok()
17382 );
17383 runner.state = RunState::load(&runner.state.id).unwrap();
17385 runner.sync_to_base().await.unwrap();
17386 assert!(runner.state.base_sync_blocked());
17387 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17388 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17389 assert_eq!(
17390 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17391 result
17392 );
17393 assert_eq!(
17394 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17395 "retained work\n"
17396 );
17397 assert!(
17398 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17399 .await
17400 .unwrap()
17401 .ok()
17402 );
17403 }
17404 }
17405
17406 #[cfg(unix)]
17407 #[tokio::test]
17408 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17409 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17410 runner.state.authorize_base_sync(&task).unwrap();
17411 assert!(runner.state.consume_base_recovery(&head));
17412 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17413 runner.state.reserve_base_operation(&head, &tip);
17414 runner.state.save().unwrap();
17415 let scratch = runner.state.dir().join("base-sync");
17416 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17417 .await
17418 .unwrap();
17419 git::sync_to_head(&runner.state.repo).await.unwrap();
17420 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17421 git::git(&runner.state.repo, &["add", "foreign"])
17422 .await
17423 .unwrap();
17424 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17425 .await
17426 .unwrap();
17427 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17428 runner.state = RunState::load(&runner.state.id).unwrap();
17429 runner.sync_to_base().await.unwrap();
17430 assert!(runner.state.base_sync_blocked());
17431 assert!(
17432 !runner
17433 .state
17434 .base_sync_recovery
17435 .operations
17436 .last()
17437 .unwrap()
17438 .finished
17439 );
17440 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17441 assert_eq!(
17442 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17443 foreign_head
17444 );
17445 }
17446
17447 #[cfg(unix)]
17448 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17449 {
17450 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17451 fresh_base_interval_fixture(&mut runner);
17452 let repo = &runner.state.repo;
17453 git::git(repo, &["checkout", "main"]).await.unwrap();
17454 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17455 git::git(repo, &["add", "retained"]).await.unwrap();
17456 git::git(repo, &["commit", "-m", "upstream retained change"])
17457 .await
17458 .unwrap();
17459 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17460 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17461 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17462 let scratch = runner.state.dir().join("base-sync");
17463 assert!(matches!(
17464 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17465 .await
17466 .unwrap(),
17467 git::RebaseStart::Conflicted(_)
17468 ));
17469 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17470 agent: "fixture".into(),
17471 paths: vec!["retained".into()],
17472 from: Some(head.clone()),
17473 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17474 index_outside: None,
17475 finished: false,
17476 error: None,
17477 launcher: None,
17478 infra: None,
17479 outage: None,
17480 });
17481 (tmp, runner, head, tip, scratch)
17482 }
17483
17484 #[cfg(unix)]
17485 #[tokio::test]
17486 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17487 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17488 std::fs::write(
17491 scratch.join("retained"),
17492 "upstream and retained resolution\n",
17493 )
17494 .unwrap();
17495 runner.state.resource_block = Some(crate::run::ResourceBlock {
17496 node: "rebase".into(),
17497 kind: "repository coordination".into(),
17498 detail: "busy".into(),
17499 at: Timestamp::now(),
17500 cause: crate::run::ResourceBlockCause::Other,
17501 });
17502 runner.state.save().unwrap();
17503 let newer = git::git(
17506 &runner.state.repo,
17507 &[
17508 "commit-tree",
17509 "main^{tree}",
17510 "-p",
17511 "main",
17512 "-m",
17513 "base advanced during wait",
17514 ],
17515 )
17516 .await
17517 .unwrap();
17518 git::git(
17519 &runner.state.repo,
17520 &[
17521 "push",
17522 "origin",
17523 &format!("{}:refs/heads/main", newer.trim()),
17524 ],
17525 )
17526 .await
17527 .unwrap();
17528 runner.state = RunState::load(&runner.state.id).unwrap();
17529 runner.state.resource_block = None;
17530 runner.sync_to_base().await.unwrap();
17531 assert!(!runner.state.base_sync_blocked());
17532 let op = &runner.state.base_sync_recovery.operations[0];
17533 assert_eq!(op.head, head);
17534 assert_eq!(op.tip, tip);
17535 assert!(op.finished);
17536 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17537 assert_eq!(runner.state.rebase_fixes.len(), 1);
17538 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17539 .await
17540 .unwrap();
17541 assert_ne!(latest, tip);
17542 runner.state = RunState::load(&runner.state.id).unwrap();
17543 runner.sync_to_base().await.unwrap();
17544 assert!(!runner.state.base_sync_blocked());
17545 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17546 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17547 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17548 }
17549
17550 #[cfg(unix)]
17551 #[tokio::test]
17552 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17553 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17554 runner.state.reserve_base_operation(&head, &tip);
17555 runner.state.save().unwrap();
17556 std::fs::write(
17557 scratch.join("retained"),
17558 "upstream and retained resolution\n",
17559 )
17560 .unwrap();
17561 git::git(&scratch, &["add", "retained"]).await.unwrap();
17562 git::git(
17563 &scratch,
17564 &["-c", "core.editor=true", "rebase", "--continue"],
17565 )
17566 .await
17567 .unwrap();
17568 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17569 assert!(
17570 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17571 .await
17572 .unwrap()
17573 .is_empty()
17574 );
17575 assert_eq!(
17576 crate::rebase::finish(
17577 &mut runner.state,
17578 &scratch,
17579 "fix/retained",
17580 &head,
17581 &tip,
17582 &["retained".into()],
17583 ""
17584 )
17585 .await
17586 .unwrap(),
17587 crate::rebase::Rebased::Applied
17588 );
17589 assert!(!scratch.exists());
17590 runner.state = RunState::load(&runner.state.id).unwrap();
17592 let op = &runner.state.base_sync_recovery.operations[0];
17593 assert!(!op.finished);
17594 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17595 runner.sync_to_base().await.unwrap();
17596 assert!(!runner.state.base_sync_blocked());
17597 assert!(runner.state.base_sync_recovery.operations[0].finished);
17598 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17599 assert_eq!(runner.state.rebase_fixes.len(), 1);
17600 let winner = runner.state.winner().unwrap().clone();
17601 let wait = runner.repository_wait();
17602 assert!(
17603 verify_current_head(&mut runner.state, &winner, &wait)
17604 .await
17605 .unwrap()
17606 );
17607 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17608 }
17609
17610 #[cfg(unix)]
17611 #[tokio::test]
17612 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17613 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17614 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17615 runner.state.reserve_base_operation(&head, &tip);
17616 runner.state.save().unwrap();
17617 std::fs::write(
17618 scratch.join("retained"),
17619 "upstream and retained resolution\n",
17620 )
17621 .unwrap();
17622 git::git(&scratch, &["add", "retained"]).await.unwrap();
17623 git::git(
17624 &scratch,
17625 &["-c", "core.editor=true", "rebase", "--continue"],
17626 )
17627 .await
17628 .unwrap();
17629 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17630 assert!(!git::rebase_in_progress(&scratch).await);
17631 assert!(
17632 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17633 .await
17634 .unwrap()
17635 .is_empty()
17636 );
17637 match damage {
17638 Some("record") => {
17639 let path =
17640 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17641 .await
17642 .unwrap();
17643 std::fs::write(scratch.join(path), "").unwrap();
17644 }
17645 Some("foreign") => {
17646 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17647 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17648 git::git(&scratch, &["commit", "-m", "foreign work"])
17649 .await
17650 .unwrap();
17651 }
17652 Some("dirty") => {
17653 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17654 }
17655 _ => {}
17656 }
17657 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17658 runner.state = RunState::load(&runner.state.id).unwrap();
17660 assert!(
17661 runner.state.base_sync_recovery.operations[0]
17662 .validated_head
17663 .is_none()
17664 );
17665 runner.sync_to_base().await.unwrap();
17666 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17667 assert_eq!(runner.state.base_episode_used(), 1);
17668 assert_eq!(runner.state.rebase_fixes.len(), 1);
17669 assert_eq!(
17670 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17671 retained_head
17672 );
17673 if damage.is_some() {
17674 assert!(runner.state.base_sync_blocked());
17675 assert!(scratch.exists());
17676 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17677 assert!(
17678 runner.state.base_sync_recovery.operations[0]
17679 .validated_head
17680 .is_none()
17681 );
17682 } else {
17683 assert!(!runner.state.base_sync_blocked());
17684 assert!(!scratch.exists());
17685 assert!(runner.state.base_sync_recovery.operations[0].finished);
17686 assert_eq!(
17687 runner.state.base_sync_recovery.operations[0]
17688 .validated_head
17689 .as_deref(),
17690 Some(result.as_str())
17691 );
17692 let winner = runner.state.winner().unwrap().clone();
17693 let wait = runner.repository_wait();
17694 assert!(
17695 verify_current_head(&mut runner.state, &winner, &wait)
17696 .await
17697 .unwrap()
17698 );
17699 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17700 runner.state = RunState::load(&runner.state.id).unwrap();
17701 runner.sync_to_base().await.unwrap();
17702 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17703 }
17704 }
17705 }
17706
17707 #[cfg(unix)]
17708 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17709 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17710 runner.state.prep_complete = Some(true);
17712 runner.state.base_sync = None;
17713 runner.state.base_sync_recovery = Default::default();
17714 runner.state.config.graph.gate_fix_rounds = 1;
17715 let mut review = review_round(true, 0, 1, 1, false, true);
17716 review.head = head.clone();
17717 review.verified_head = Some(head.clone());
17718 runner.state.reviews = vec![review];
17719 runner.state.gate = vec![CommandOutcome {
17720 command: "old gate".into(),
17721 code: Some(1),
17722 output_tail: "old failure".into(),
17723 duration_ms: 0,
17724 resource_blocked: false,
17725 }];
17726 runner.state.gate_head = Some(head.clone());
17727 runner.state.gate_ran = true;
17728 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17729 agent: "fixture".into(),
17730 failed: runner.state.gate.clone(),
17731 notes: "spent".into(),
17732 committed: false,
17733 error: None,
17734 }];
17735 runner.state.land_approval = Some(crate::run::LandApproval {
17736 question: "old approval".into(),
17737 head: head.clone(),
17738 });
17739 runner.state.status = RunStatus::Blocked;
17740 (tmp, runner, head)
17741 }
17742
17743 #[cfg(unix)]
17744 #[tokio::test]
17745 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17746 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17747 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17748 let phases = tmp.path().join("phases");
17749 runner.state.config.verify.e2e = vec![format!(
17750 "echo e2e >> '{}'; {}",
17751 phases.display(),
17752 if e2e_ok {
17753 "true"
17754 } else {
17755 "echo new-e2e-failure; false"
17756 }
17757 )];
17758 runner.state.config.verify.gate = vec![format!(
17759 "echo gate >> '{}'; {}",
17760 phases.display(),
17761 if gate_ok {
17762 "true"
17763 } else {
17764 "echo new-gate-failure; false"
17765 }
17766 )];
17767 runner.state.save().unwrap();
17768 runner.state = RunState::load(&runner.state.id).unwrap();
17769 runner.sync_to_base().await.unwrap();
17770 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17771 assert_ne!(head, old);
17772 runner.state = RunState::load(&runner.state.id).unwrap();
17774 runner.execute_graph().await.unwrap();
17775 assert_eq!(runner.state.gate_fixes.len(), 1);
17776 assert_eq!(runner.state.reviews.len(), 1);
17777 assert_eq!(runner.state.reviews[0].head, old);
17778 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17779 assert_eq!(runner.state.verification_history.len(), 1);
17780 let history = &runner.state.verification_history[0];
17781 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17782 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17783 assert!(history.archived_at.is_some());
17784 assert_eq!(history.fixes_spent, Some(1));
17785 assert_eq!(history.gate[0].output_tail, "old failure");
17786 let archive_events: Vec<_> = runner
17787 .state
17788 .events
17789 .iter()
17790 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17791 .collect();
17792 assert_eq!(archive_events.len(), 1);
17793 assert!(archive_events[0].message.contains(&short(&old)));
17794 assert!(archive_events[0].message.contains(&short(&head)));
17795 assert!(
17796 archive_events[0]
17797 .message
17798 .contains("1 gate-fix round(s) already spent stay spent")
17799 );
17800 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17801 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17802 if e2e_ok {
17803 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17804 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17805 } else {
17806 assert!(!runner.state.gate_ran);
17807 }
17808 assert_eq!(
17809 runner.state.status,
17810 if e2e_ok && gate_ok {
17811 RunStatus::Ready
17812 } else {
17813 RunStatus::Blocked
17814 }
17815 );
17816 if !e2e_ok || !gate_ok {
17817 let cause = runner.state.verification_blocking_cause().unwrap();
17818 assert!(cause.contains(&head));
17819 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17820 runner.state.event("land", "rebased successfully");
17822 let notice = crate::notices::run_ended(&runner.state).unwrap();
17823 assert!(!format!("{notice:?}").contains("rebased successfully"));
17824 assert!(runner.state.merge.is_none());
17825 }
17826 let before = std::fs::read_to_string(&phases).unwrap();
17827 runner.state.save().unwrap();
17828 runner.state = RunState::load(&runner.state.id).unwrap();
17829 runner.review_loop().await.unwrap();
17830 runner.gate().await.unwrap();
17831 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17832 assert_eq!(runner.state.verification_history.len(), 1);
17833 let history = &runner.state.verification_history[0];
17834 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17835 assert!(history.archived_at.is_some());
17836 assert_eq!(history.fixes_spent, Some(1));
17837 }
17838 }
17839
17840 #[cfg(unix)]
17841 #[tokio::test]
17842 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17843 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17844 let marker = tmp.path().join("gate-ran");
17845 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17846 runner.review_loop().await.unwrap();
17847 runner.gate().await.unwrap();
17848 assert_eq!(runner.state.status, RunStatus::Blocked);
17849 assert!(!marker.exists());
17850 assert!(runner.state.verification_history.is_empty());
17851 runner.state.gate_head = None;
17852 runner.gate().await.unwrap();
17853 assert!(marker.exists());
17854 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17855 assert_eq!(runner.state.verification_history.len(), 1);
17856 assert_eq!(runner.state.gate_fixes.len(), 1);
17857 }
17858
17859 #[cfg(unix)]
17860 #[tokio::test]
17861 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17862 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17863 runner.sync_to_base().await.unwrap();
17864 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17865 runner.state.reconcile_verification_head(&head);
17866 for outcomes in [
17868 Vec::new(),
17869 vec![CommandOutcome {
17870 command: "true".into(),
17871 code: None,
17872 output_tail: "cache unavailable".into(),
17873 duration_ms: 0,
17874 resource_blocked: true,
17875 }],
17876 ] {
17877 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17878 head: head.clone(),
17879 passed: false,
17880 outcomes,
17881 });
17882 runner.state.gate_ran = false;
17883 runner.state.gate.clear();
17884 runner.state.save().unwrap();
17885 runner.state = RunState::load(&runner.state.id).unwrap();
17886 runner.merge().await.unwrap();
17887 assert!(runner.state.merge.is_none());
17888 runner.review_loop().await.unwrap();
17889 runner.gate().await.unwrap();
17890 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17891 assert!(runner.state.gate_allows_progress());
17892 assert_eq!(runner.state.gate_fixes.len(), 1);
17893 assert!(runner.state.merge.is_none());
17894 }
17895 git::git(
17897 &runner.state.repo,
17898 &["commit", "--allow-empty", "-m", "external update"],
17899 )
17900 .await
17901 .unwrap();
17902 runner.merge().await.unwrap();
17903 assert!(runner.state.merge.is_none());
17904 assert_eq!(runner.state.status, RunStatus::Blocked);
17905 }
17906
17907 #[cfg(unix)]
17908 #[tokio::test]
17909 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17910 for waiting_on_gate in [false, true] {
17911 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17912 runner.sync_to_base().await.unwrap();
17913 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17914 runner.state.reconcile_verification_head(&head);
17915 if waiting_on_gate {
17916 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17917 head: head.clone(),
17918 passed: true,
17919 outcomes: Vec::new(),
17920 });
17921 }
17922 let cache = tmp.path().join("cache");
17923 runner.state.config.verify.e2e =
17924 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17925 runner.state.config.verify.gate = vec!["true".into()];
17926 runner.state.config.graph.timeout_resource_wait = 1;
17927 let owner =
17928 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17929 let guard =
17930 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17931 crate::cache::AcquireOutcome::Acquired(g) => g,
17932 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17933 };
17934 runner.review_loop().await.unwrap();
17935 runner.gate().await.unwrap();
17936 assert!(!runner.state.gate_ran);
17937 assert_eq!(runner.state.gate_fixes.len(), 1);
17938 assert!(runner.state.resource_block.is_some());
17939 runner.merge().await.unwrap();
17940 assert!(runner.state.merge.is_none());
17941 drop(guard);
17942 runner.state = RunState::load(&runner.state.id).unwrap();
17943 runner.state.resource_block = None;
17945 runner.review_loop().await.unwrap();
17946 runner.gate().await.unwrap();
17947 assert!(runner.state.gate_allows_progress());
17948 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17949 assert_eq!(runner.state.gate_fixes.len(), 1);
17950 }
17951 }
17952
17953 #[cfg(unix)]
17954 #[tokio::test]
17955 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17956 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17957 runner.sync_to_base().await.unwrap();
17958 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17959 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17960 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17962 runner.review_loop().await.unwrap();
17963 runner.gate().await.unwrap();
17964 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17965 assert_ne!(before, after);
17966 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17967 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17968 assert!(!runner.state.gate_allows_progress());
17969 runner.merge().await.unwrap();
17970 assert!(runner.state.merge.is_none());
17971 assert!(
17972 runner
17973 .state
17974 .verification_history
17975 .iter()
17976 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17977 );
17978
17979 runner.state.reviews[0].clean = false;
17981 runner.state.reviews[0].answered = 0;
17982 runner.state.config.verify.e2e = vec!["true".into()];
17983 runner.review_loop().await.unwrap();
17984 runner.gate().await.unwrap();
17985 runner.merge().await.unwrap();
17986 assert_eq!(runner.state.status, RunStatus::Blocked);
17987 assert!(runner.state.merge.is_none());
17988 assert!(
17989 runner
17990 .state
17991 .verification_blocking_cause()
17992 .unwrap()
17993 .contains("review blocked")
17994 );
17995 }
17996
17997 #[cfg(unix)]
17998 #[tokio::test]
17999 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18000 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18001 runner.state.gate_head = None;
18002 runner.state.config.verify.gate =
18003 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18004 runner.review_loop().await.unwrap();
18005 runner.gate().await.unwrap();
18006 assert_ne!(
18007 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18008 old
18009 );
18010 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18011 assert!(!runner.state.gate_allows_progress());
18012 assert_eq!(runner.state.status, RunStatus::Blocked);
18013 runner.merge().await.unwrap();
18014 assert!(runner.state.merge.is_none());
18015 }
18016
18017 #[cfg(unix)]
18018 #[tokio::test]
18019 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18020 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18021 let mut old_review = review_round(true, 0, 1, 1, false, true);
18022 old_review.head = head.clone();
18023 old_review.verified_head = Some(head.clone());
18024 runner.state.reviews = vec![old_review];
18025 runner.state.gate = vec![CommandOutcome {
18026 command: "old gate".into(),
18027 code: None,
18028 output_tail: "timed out after 2400s".into(),
18029 duration_ms: 2_400_000,
18030 resource_blocked: false,
18031 }];
18032 runner.state.gate_ran = true;
18033 runner.state.gate_head = Some(head.clone());
18034 runner.state.land_approval = Some(crate::run::LandApproval {
18035 question: "old-head-approval".into(),
18036 head: head.clone(),
18037 });
18038 runner.state.authorize_base_sync(&task).unwrap();
18039 runner.sync_to_base().await.unwrap();
18040 assert!(!runner.state.base_sync_blocked());
18041 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18042 assert_ne!(head, fresh);
18043 assert_eq!(runner.state.reviews.len(), 1);
18044 assert!(!runner.state.gate_ran);
18045 let saved = &runner.state.base_sync_recovery.authorizations[0];
18046 assert_eq!(saved.reviews[0].head, head);
18047 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18048 assert!(saved.gate_ran);
18049 assert!(!saved.gate[0].ok());
18050 runner.review_loop().await.unwrap();
18051 runner.gate().await.unwrap();
18052 assert_eq!(runner.state.reviews[0].head, head);
18053 assert_eq!(
18054 runner.state.reviews[0].verified_head.as_deref(),
18055 Some(head.as_str())
18056 );
18057 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18058 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18059 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18060 assert!(runner.state.gate_ran);
18061 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18062 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18063 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18064 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18065 assert_eq!(
18066 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18067 "retained change"
18068 );
18069 }
18070
18071 #[cfg(unix)]
18072 #[tokio::test]
18073 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18074 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18075 let queue = crate::queue::Queue::open();
18076 let mut owner = queue.get(&task).unwrap();
18077 owner.release_operator("test-resume", None);
18078 queue.put(&mut owner).unwrap();
18079 let generation = owner.resumption.as_ref().unwrap().generation;
18080 runner.state.task_instruction = Some(owner.instruction.clone());
18081 runner.state.driver_exited = true;
18082 runner.state.save().unwrap();
18083 let home = crate::run::home();
18084 let _claim = queue.claim(&task).unwrap();
18085 owner.resumption.as_mut().unwrap().at =
18086 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18087 queue.put(&mut owner).unwrap();
18088 assert!(
18089 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18090 .await
18091 .is_err(),
18092 "an approval from before the stop grants nothing"
18093 );
18094 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18095 queue.put(&mut owner).unwrap();
18096 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18097 .await
18098 .unwrap();
18099 assert!(
18100 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18101 .await
18102 .is_err()
18103 );
18104 runner.state = RunState::load(&runner.state.id).unwrap();
18105 assert_eq!(
18106 runner.state.base_sync_recovery.authorizations[0].generation,
18107 Some(generation)
18108 );
18109 runner.sync_to_base().await.unwrap();
18110 let repo = runner.state.repo.clone();
18111 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18112 git::git(&repo, &["checkout", "main"]).await.unwrap();
18113 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18114 git::git(&repo, &["add", "."]).await.unwrap();
18115 git::git(&repo, &["commit", "-m", "move base again"])
18116 .await
18117 .unwrap();
18118 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18119 git::git(&repo, &["checkout", "fix/retained"])
18120 .await
18121 .unwrap();
18122 runner.sync_to_base().await.unwrap();
18123 assert!(!runner.state.base_sync_blocked());
18124 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18125 assert_eq!(runner.state.base_episode_used(), 2);
18126 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18127 assert_ne!(fresh, head);
18128 assert!(!runner.state.base_recovery_pending());
18129 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18131 queue.put(&mut owner).unwrap();
18132 assert!(
18133 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18134 .await
18135 .is_err()
18136 );
18137 runner.state = RunState::load(&runner.state.id).unwrap();
18138 runner.sync_to_base().await.unwrap();
18139 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18140 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18141 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18142 }
18143
18144 #[cfg(unix)]
18145 #[tokio::test]
18146 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18147 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18148 runner.state.base_sync = None;
18149 runner.state.status = RunStatus::Blocked;
18150 let mut second = runner.state.candidates[0].clone();
18151 second.label = 'B';
18152 runner.state.candidates.push(second);
18153 assert!(runner.state.votes.is_empty());
18154 assert!(runner.state.deliberation.is_empty());
18155 runner.deliberate().await.unwrap();
18156 runner.vote().await.unwrap();
18157 assert_eq!(runner.state.status, RunStatus::Blocked);
18158 assert!(runner.state.votes.is_empty());
18159 assert!(runner.state.deliberation.is_empty());
18160 }
18161
18162 fn runner_at(status: RunStatus) -> Runner {
18165 let mut state = RunState::new(
18166 PathBuf::from("/nonexistent/repo"),
18167 "main".to_owned(),
18168 "deadbeef".to_owned(),
18169 "task".to_owned(),
18170 Config::default(),
18171 );
18172 state.config.disk.min_free_bytes = 0;
18174 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18177 state.id = format!(
18178 "20000101-000000-{:04x}",
18179 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18180 );
18181 state.status = status;
18182 Runner {
18183 state,
18184 roles: ResolvedRoles {
18185 implementers: Vec::new(),
18186 judges: Vec::new(),
18187 reviewers: Vec::new(),
18188 fixer: None,
18189 conductor: conductor(),
18190 implementer_roster: Vec::new(),
18191 judge_roster: Vec::new(),
18192 reviewer_roster: Vec::new(),
18193 },
18194 sem: Arc::new(Semaphore::new(1)),
18195 pause: Pause::new(),
18196 interrupt: Pause::new(),
18197 }
18198 }
18199
18200 #[tokio::test]
18201 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18202 crate::run::pin_test_home();
18203 let mut runner = runner_at(RunStatus::Gating);
18204 runner.state.save().unwrap();
18205 let id = runner.state.id.clone();
18206 let passes = runner.state.passes.len();
18207 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18208 let mut claiming = Box::pin(runner.claim_driver());
18209 std::future::poll_fn(|cx| {
18210 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18211 std::task::Poll::Ready(())
18212 })
18213 .await;
18214 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18215 drop(owner);
18216 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18217 .await
18218 .unwrap()
18219 .unwrap();
18220 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18221 assert!(execution.runner.state.resource_block.is_none());
18222 assert_eq!(
18223 execution.runner.state.liveness(false),
18224 crate::run::Liveness::Live
18225 );
18226 drop(execution);
18227 assert!(RunState::load(&id).unwrap().driver_exited);
18228 }
18229
18230 #[tokio::test]
18231 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18232 crate::run::pin_test_home();
18233 let mut runner = runner_at(RunStatus::Gating);
18234 runner.state.save().unwrap();
18235 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18238 assert!(runner.claim_driver().await.is_err());
18239 assert!(runner.state.driver_exited);
18240 assert_eq!(runner.state.driver_pid, None);
18241 let disk = RunState::load(&runner.state.id).unwrap();
18242 assert!(disk.driver_exited);
18243 assert_eq!(disk.driver_pid, None);
18244 assert_eq!(disk.status, RunStatus::Gating);
18245 }
18246
18247 #[tokio::test]
18248 async fn an_early_graph_error_releases_its_driver() {
18249 crate::run::pin_test_home();
18250 let mut runner = runner_at(RunStatus::Blocked);
18251 runner.state.config.graph.selected_implementer = Some("selected".into());
18252 runner.state.save().unwrap();
18253 assert!(runner.execute().await.is_err());
18254 let disk = RunState::load(&runner.state.id).unwrap();
18255 assert!(disk.driver_exited);
18256 assert_eq!(disk.driver_pid, None);
18257 assert_eq!(disk.status, RunStatus::Blocked);
18258 }
18259
18260 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18261 crate::run::MergeRepair {
18262 question: "repair-question".into(),
18263 head: "old-head".into(),
18264 pr: "https://github.com/o/r/pull/42".into(),
18265 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18266 gate: vec![outcome(Some(0), false)],
18267 gate_head: Some("old-head".into()),
18268 gate_fixes: Vec::new(),
18269 pre_gate: Vec::new(),
18270 phase,
18271 failure: None,
18272 }
18273 }
18274
18275 #[tokio::test]
18276 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18277 crate::run::pin_test_home();
18278 let mut runner = runner_at(RunStatus::Landing);
18279 runner.state.merge_repairs.push(merge_repair_request(
18280 crate::run::MergeRepairPhase::Requested,
18281 ));
18282 runner.state.save().unwrap();
18283 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18284 assert!(
18285 runner
18286 .execute()
18287 .await
18288 .unwrap_err()
18289 .to_string()
18290 .contains("already running")
18291 );
18292 assert_eq!(
18293 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18294 crate::run::MergeRepairPhase::Requested
18295 );
18296 drop(claim);
18297 let mut disk = runner.state.clone();
18298 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18299 disk.save().unwrap();
18300 assert!(
18301 runner
18302 .execute()
18303 .await
18304 .unwrap_err()
18305 .to_string()
18306 .contains("decision changed")
18307 );
18308 assert_eq!(
18309 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18310 crate::run::MergeRepairPhase::Complete
18311 );
18312 }
18313
18314 #[test]
18315 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18316 crate::run::pin_test_home();
18317 let mut runner = runner_at(RunStatus::Landing);
18318 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18319 runner.state.reviews = request.reviews.clone();
18320 runner.state.gate = request.gate.clone();
18321 runner.state.gate_ran = true;
18322 runner.state.land_approval = Some(crate::run::LandApproval {
18323 question: "old".into(),
18324 head: "old-head".into(),
18325 });
18326 runner.state.merge_repairs.push(request);
18327 runner.begin_merge_repair_review(0).unwrap();
18328 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18329 assert!(!runner.state.gate_status().ok());
18330 assert!(runner.state.land_approval.is_none());
18331 let disk = RunState::load(&runner.state.id).unwrap();
18332 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18333 assert!(disk.merge_repairs[0].gate[0].ok());
18334 assert_eq!(
18335 disk.merge_repairs[0].phase,
18336 crate::run::MergeRepairPhase::Reviewing
18337 );
18338 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18339 assert!(disk.reviews.is_empty());
18340 }
18341
18342 #[tokio::test]
18343 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18344 crate::run::pin_test_home();
18345 for phase in [
18346 crate::run::MergeRepairPhase::Fixing,
18347 crate::run::MergeRepairPhase::Requested,
18348 ] {
18349 let mut runner = runner_at(RunStatus::Landing);
18350 runner.state.config.graph.land_rounds = 0;
18351 runner.state.merge_repairs.push(merge_repair_request(phase));
18352 runner.state.save().unwrap();
18353 assert!(!runner.repair_merge_review().await.unwrap());
18354 assert_eq!(runner.state.status, RunStatus::Blocked);
18355 assert_eq!(
18356 runner.state.merge_repairs[0].phase,
18357 crate::run::MergeRepairPhase::Failed
18358 );
18359 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18360 assert!(
18361 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18362 "interrupted"
18363 } else {
18364 "budget exhausted"
18365 })
18366 );
18367 assert!(runner.state.merge.is_none());
18368 }
18369 }
18370
18371 #[test]
18375 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18376 crate::run::pin_test_home();
18377 let mut runner = runner_at(RunStatus::Implementing);
18378 let interrupt = Pause::new();
18379 runner.watch_interrupt(interrupt.clone());
18380
18381 interrupt.park_because("task a1b2 asked to run first");
18382
18383 assert!(runner.park_here().expect("park_here"));
18384 assert!(runner.state.parked);
18385 let last = runner.state.events.last().expect("a park event");
18386 assert_eq!(last.node, "park");
18387 assert!(
18388 last.message.contains("task a1b2 asked to run first"),
18389 "expected the interrupt reason in {:?}",
18390 last.message
18391 );
18392 }
18393
18394 #[test]
18397 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18398 crate::run::pin_test_home();
18399 let mut runner = runner_at(RunStatus::Implementing);
18400 runner.state.begin_pass();
18401 let shutdown = Pause::new();
18402 runner.on_pause(shutdown.clone());
18403 let upgrade = ParkCause::Upgrade {
18404 from: "0.121.0".to_owned(),
18405 to: Some("0.121.1".to_owned()),
18406 };
18407 shutdown.park_for(upgrade.clone());
18408 shutdown.park_for(ParkCause::Operator);
18409
18410 assert!(runner.park_here().expect("park_here"));
18411 assert_eq!(
18412 runner.state.passes[0].end,
18413 Some(crate::run::PassEnd::Parked { cause: upgrade })
18414 );
18415 let last = runner.state.events.last().expect("a park event");
18416 assert!(last.message.contains("0.121.0"), "{}", last.message);
18417 }
18418
18419 #[tokio::test]
18420 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18421 crate::run::pin_test_home();
18422 let mut runner = runner_at(RunStatus::Reviewing);
18423 runner.state.save().unwrap();
18424 let mut execution = runner.claim_driver().await.expect("first execution");
18425 assert_eq!(execution.runner.state.passes.len(), 1);
18426 execution
18427 .driver
18428 .finish(&mut execution.runner.state)
18429 .await
18430 .unwrap();
18431 }
18432
18433 #[tokio::test]
18434 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18435 crate::run::pin_test_home();
18436 let mut runner = runner_at(RunStatus::Reviewing);
18437 runner.state.config.agents = vec![conductor()];
18438 runner.state.config.graph.timeout_resource_wait = 0;
18439 runner.state.save().unwrap();
18440 let id = runner.state.id.clone();
18441 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18442 assert!(runner.claim_driver().await.is_err());
18443 drop(owner);
18444 drop(runner);
18445 let mut resumed = Runner::resume(&id).unwrap();
18446 assert_eq!(resumed.state.driver_claimed, Some(false));
18447 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18448 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18449 let mut execution = resumed
18450 .claim_driver()
18451 .await
18452 .expect("retry initial claim after reload");
18453 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18454 assert_eq!(execution.runner.state.passes.len(), 1);
18455 execution
18456 .driver
18457 .finish(&mut execution.runner.state)
18458 .await
18459 .unwrap();
18460 }
18461
18462 #[tokio::test]
18463 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18464 crate::run::pin_test_home();
18465 let mut runner = runner_at(RunStatus::Reviewing);
18466 runner.state.config.agents = vec![conductor()];
18467 runner.state.save().unwrap();
18468 let id = runner.state.id.clone();
18469 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18470 let mut resumed = Runner::resume(&id).unwrap();
18471 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18472 assert!(resumed.claim_driver().await.is_err());
18473 assert!(RunState::load(&id).unwrap().passes.is_empty());
18474 drop(lease);
18475 let mut execution = resumed.claim_driver().await.unwrap();
18476 execution
18477 .driver
18478 .finish(&mut execution.runner.state)
18479 .await
18480 .unwrap();
18481 }
18482
18483 #[tokio::test]
18484 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18485 crate::run::pin_test_home();
18486 let mut runner = runner_at(RunStatus::Reviewing);
18487 let state = &mut runner.state;
18488 state.config.agents = vec![conductor()];
18489 state.driver_claimed = None;
18490 state.driver_pid = Some(std::process::id());
18491 state.driver_started_at = None;
18492 state.begin_pass();
18493 state.save().unwrap();
18494 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18495 std::fs::write(
18496 state.dir().join("driver.json"),
18497 serde_json::to_vec(&serde_json::json!({
18498 "pid": state.driver_pid, "started_at": null,
18499 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18500 }))
18501 .unwrap(),
18502 )
18503 .unwrap();
18504 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18505 drop(lease);
18506 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18507 let mut resumed = Runner::resume(&state.id).unwrap();
18508 assert!(!resumed.state.driver_exited);
18509 let mut execution = resumed
18510 .claim_driver()
18511 .await
18512 .expect("lease proves previous execution ended");
18513 assert_eq!(execution.runner.state.passes.len(), 2);
18514 execution
18515 .driver
18516 .finish(&mut execution.runner.state)
18517 .await
18518 .unwrap();
18519 }
18520
18521 #[tokio::test]
18522 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18523 crate::run::pin_test_home();
18524 let mut runner = runner_at(RunStatus::Reviewing);
18525 let mut json = serde_json::to_value(&runner.state).unwrap();
18526 json.as_object_mut().unwrap().remove("driver_claimed");
18527 runner.state = serde_json::from_value(json).unwrap();
18528 assert_eq!(runner.state.driver_claimed, None);
18529 runner.state.save().unwrap();
18530 let mut execution = runner
18531 .claim_driver()
18532 .await
18533 .expect("the execution lease, not a missing marker, excludes a second driver");
18534 assert_eq!(execution.runner.state.passes.len(), 1);
18535 execution
18536 .driver
18537 .finish(&mut execution.runner.state)
18538 .await
18539 .unwrap();
18540 }
18541
18542 #[tokio::test]
18543 async fn driver_admission_refuses_a_proven_live_driver() {
18544 crate::run::pin_test_home();
18545 let mut runner = runner_at(RunStatus::Reviewing);
18546 let pid = std::process::id();
18547 runner.state.driver_pid = Some(pid);
18548 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18549 assert!(runner.state.driver_started_at.is_some());
18550 runner.state.save().unwrap();
18551 let error = runner
18552 .claim_driver()
18553 .await
18554 .err()
18555 .expect("a live driver must refuse");
18556 assert!(error.to_string().contains("live driver"), "{error:#}");
18557 let saved = RunState::load(&runner.state.id).unwrap();
18558 assert_eq!(saved.driver_pid, Some(pid));
18559 assert_eq!(saved.status, RunStatus::Reviewing);
18560 assert!(saved.passes.is_empty());
18561 }
18562
18563 #[test]
18564 fn parking_resource_halt_persists_the_pass_once() {
18565 crate::run::pin_test_home();
18566 let mut runner = runner_at(RunStatus::Reviewing);
18567 runner.state.begin_pass();
18568 runner.state.resource_block = Some(crate::run::ResourceBlock {
18569 node: "fix".into(),
18570 kind: "shared build cache".into(),
18571 detail: "cancelled".into(),
18572 at: Timestamp::now(),
18573 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18574 });
18575 let cause = ParkCause::Upgrade {
18576 from: "old".into(),
18577 to: Some("new".into()),
18578 };
18579 runner.pause.park_for(cause.clone());
18580 runner.stop_for_resource().unwrap();
18583 assert!(runner.resource_halt().unwrap());
18584 assert!(runner.park_here().unwrap());
18585 let saved = RunState::load(&runner.state.id).unwrap();
18586 assert!(saved.parked);
18587 assert_eq!(saved.status, RunStatus::Reviewing);
18588 assert!(saved.passes[0].ended_at.is_some());
18589 assert_eq!(
18590 saved.passes[0].end,
18591 Some(crate::run::PassEnd::Parked {
18592 cause: cause.clone()
18593 })
18594 );
18595 assert_eq!(
18596 saved.resource_block.unwrap().cause,
18597 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18598 );
18599 assert_eq!(
18600 saved
18601 .events
18602 .iter()
18603 .filter(|event| event.node == "park")
18604 .count(),
18605 1
18606 );
18607 }
18608
18609 #[test]
18617 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18618 crate::run::pin_test_home();
18619 let mut runner = runner_at(RunStatus::Implementing);
18620 let shutdown = Pause::new();
18621 runner.on_pause(shutdown.clone());
18622 let interrupt = Pause::new();
18623 runner.watch_interrupt(interrupt.clone());
18624
18625 assert!(!runner.park_here().expect("park_here"));
18627 assert!(!runner.state.parked);
18628
18629 interrupt.park_because("test");
18631 assert!(!shutdown.parked());
18632 assert!(runner.park_here().expect("park_here"));
18633 }
18634
18635 #[tokio::test]
18649 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18650 crate::run::pin_test_home();
18651 let mut runner = runner_at(RunStatus::Implementing);
18652 let interrupt = Pause::new();
18653 runner.watch_interrupt(interrupt.clone());
18654
18655 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18656 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18657
18658 let node = async move {
18662 started_tx.send(()).expect("send started");
18663 finish_rx.await.expect("recv finish");
18664 "node finished"
18665 };
18666
18667 let interrupter = async move {
18668 started_rx.await.expect("recv started");
18669 interrupt.park_because("higher-priority task waiting");
18671 tokio::task::yield_now().await;
18675 finish_tx.send(()).expect("send finish");
18676 };
18677
18678 let (node_result, ()) = tokio::join!(node, interrupter);
18679 assert_eq!(
18680 node_result, "node finished",
18681 "the in-flight call ran to completion"
18682 );
18683
18684 assert!(runner.park_here().expect("park_here"));
18687 assert!(runner.state.parked);
18688 }
18689
18690 #[test]
18696 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18697 crate::run::pin_test_home();
18698 let mut runner = runner_at(RunStatus::Judging);
18699 runner.state.config.agents = vec![conductor()];
18703 runner.state.candidates = vec![Candidate {
18704 index: 0,
18705 label: 'A',
18706 agent: "alpha".to_owned(),
18707 branch: "magi/x/A".to_owned(),
18708 worktree: PathBuf::from("/nonexistent/worktree"),
18709 summary: "did the thing".to_owned(),
18710 stat: "1 file changed".to_owned(),
18711 files: 1,
18712 commits: 1,
18713 empty: false,
18714 failed: None,
18715 verified_noop: None,
18716 duration_ms: 1234,
18717 folded: false,
18718 }];
18719 let run_id = runner.state.id.clone();
18720
18721 let interrupt = Pause::new();
18722 runner.watch_interrupt(interrupt.clone());
18723 interrupt.park_because("task c3d4 asked to run first");
18724 assert!(runner.park_here().expect("park_here"));
18725
18726 let resumed = Runner::resume(&run_id).expect("resume");
18727 assert_eq!(resumed.state.candidates.len(), 1);
18728 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18729 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18730 assert_eq!(resumed.state.status, runner.state.status);
18731 assert!(
18732 resumed.state.parked,
18733 "still parked until `execute` actually walks the graph again"
18734 );
18735 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18736 }
18737
18738 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18740 let mut q = ask::Question::new(
18741 run.to_owned(),
18742 "implement".to_owned(),
18743 "impl-A".to_owned(),
18744 "Which storage backend should the cache use?".to_owned(),
18745 String::new(),
18746 vec!["SQLite".to_owned(), "Redis".to_owned()],
18747 );
18748 store.put(&mut q).unwrap();
18749 q
18750 }
18751
18752 #[test]
18753 fn a_failed_runs_open_question_is_abandoned() {
18754 ask_test_home();
18755 let store = ask::Questions::open();
18756 let mut runner = runner_at(RunStatus::Failed);
18757 let run = runner.state.id.clone();
18758 let q = ask_open_question(&store, &run);
18759
18760 runner.settle_questions();
18761
18762 let back = store.get(&q.id).unwrap();
18763 assert!(
18764 !back.status.open(),
18765 "the seat that asked died with the run; nobody is left to read an answer"
18766 );
18767 assert!(
18768 back.detail.contains(&run) && back.detail.contains("failed"),
18769 "the reason names what the run became, not just that it is gone: {}",
18770 back.detail
18771 );
18772 }
18773
18774 #[test]
18775 fn a_merged_runs_open_question_is_abandoned_too() {
18776 ask_test_home();
18777 let store = ask::Questions::open();
18778 for status in [RunStatus::Merged, RunStatus::Ready] {
18781 let mut runner = runner_at(status);
18782 let run = runner.state.id.clone();
18783 let q = ask_open_question(&store, &run);
18784
18785 runner.settle_questions();
18786
18787 let back = store.get(&q.id).unwrap();
18788 assert!(
18789 !back.status.open(),
18790 "{status:?} run's question must not outlive the run"
18791 );
18792 }
18793 }
18794
18795 #[test]
18796 fn a_still_resumable_runs_open_question_is_left_alone() {
18797 ask_test_home();
18798 let store = ask::Questions::open();
18799 for status in [RunStatus::Blocked, RunStatus::Stalled] {
18805 let mut runner = runner_at(status);
18806 let run = runner.state.id.clone();
18807 let q = ask_open_question(&store, &run);
18808
18809 runner.settle_questions();
18810
18811 let back = store.get(&q.id).unwrap();
18812 assert!(
18813 back.status.open(),
18814 "{status:?} is still alive; the question must still be waiting"
18815 );
18816 }
18817 }
18818
18819 #[test]
18820 fn settle_questions_never_touches_an_already_answered_question() {
18821 ask_test_home();
18822 let store = ask::Questions::open();
18823 let mut runner = runner_at(RunStatus::Failed);
18824 let run = runner.state.id.clone();
18825 let mut q = ask_open_question(&store, &run);
18826 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18827 .unwrap();
18828 store.put(&mut q).unwrap();
18829
18830 runner.settle_questions();
18835 runner.settle_questions();
18836
18837 let back = store.get(&q.id).unwrap();
18838 assert_eq!(
18839 back.status,
18840 ask::QuestionStatus::Answered,
18841 "a real answer is a decision on record, never overwritten by a sweep"
18842 );
18843 }
18844
18845 #[tokio::test]
18856 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
18857 crate::run::pin_test_home();
18858 let tmp = tempfile::tempdir().expect("tempdir");
18859 let repo = tmp.path().join("repo");
18860 std::fs::create_dir_all(&repo).unwrap();
18861 init_repo(&repo);
18862
18863 let mut config = Config::default();
18864 config.graph.worktree_root = Some(tmp.path().join("wt"));
18865
18866 let mut state = RunState::new(
18867 repo.clone(),
18868 "main".to_owned(),
18869 "deadbeef".to_owned(),
18870 "task".to_owned(),
18871 config,
18872 );
18873 let root = state.worktree_root();
18874 let wt_a = root.join("cand-A");
18875 let wt_b = root.join("cand-B");
18876 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
18877 .await
18878 .expect("worktree A");
18879 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
18880 .await
18881 .expect("worktree B");
18882
18883 state.candidates = vec![
18884 Candidate {
18885 index: 0,
18886 label: 'A',
18887 agent: "alpha".to_owned(),
18888 branch: "magi/x/A".to_owned(),
18889 worktree: wt_a.clone(),
18890 summary: String::new(),
18891 stat: String::new(),
18892 files: 0,
18893 commits: 0,
18894 empty: false,
18895 failed: None,
18896 verified_noop: None,
18897 duration_ms: 0,
18898 folded: false,
18899 },
18900 Candidate {
18901 index: 1,
18902 label: 'B',
18903 agent: "beta".to_owned(),
18904 branch: "magi/x/B".to_owned(),
18905 worktree: wt_b.clone(),
18906 summary: String::new(),
18907 stat: String::new(),
18908 files: 0,
18909 commits: 0,
18910 empty: false,
18911 failed: None,
18912 verified_noop: None,
18913 duration_ms: 0,
18914 folded: false,
18915 },
18916 ];
18917 state.tally = Some(Tally {
18918 first_choice: BTreeMap::from([('A', 1)]),
18919 borda: BTreeMap::new(),
18920 winner: 'A',
18921 rankings: 1,
18922 unanimous_initial: true,
18923 deliberated: false,
18924 changed_votes: 0,
18925 unanimous_final: true,
18926 tie_break: None,
18927 judges: 1,
18928 present: 1,
18929 quorum: 1,
18930 met_quorum: true,
18931 uncontested: None,
18932 });
18933 state.status = RunStatus::Ready;
18934
18935 fold_run(&mut state, false, &crate::run::home())
18936 .await
18937 .expect("fold_run");
18938
18939 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
18940 assert!(
18941 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
18942 "the unmerged winner's branch survives"
18943 );
18944 assert!(
18945 !state.candidates[0].folded,
18946 "the winner is not marked folded"
18947 );
18948
18949 assert!(!wt_b.exists(), "the loser's worktree is removed");
18950 assert!(
18951 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
18952 "the loser's branch is removed"
18953 );
18954 assert!(state.candidates[1].folded, "the loser is marked folded");
18955 }
18956
18957 #[tokio::test]
18958 #[cfg(unix)]
18959 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
18960 use std::os::fd::AsRawFd as _;
18961 use std::time::Duration;
18962
18963 let tmp = tempfile::tempdir().unwrap();
18964 let repo = tmp.path().join("repo");
18965 std::fs::create_dir_all(&repo).unwrap();
18966 init_repo(&repo);
18967 let common = repo.join(".git");
18968 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18969 let home = tmp.path().join("home");
18970 let mut config = Config::default();
18971 config.coordination = crate::config::test_coordination();
18972 config.graph.worktree_root = Some(tmp.path().join("wt"));
18973 config.graph.timeout_resource_wait = 1;
18974 let mut state = RunState::new(
18975 repo.clone(),
18976 "main".into(),
18977 "deadbeef".into(),
18978 "task".into(),
18979 config,
18980 );
18981 let root = state.worktree_root();
18982 let candidate = root.join("cand-A");
18983 let reviewer = root.join("review-1");
18984 let branch = state.branch_for('A');
18985 git::worktree_add_branch(&repo, &candidate, &branch, "main")
18986 .await
18987 .unwrap();
18988 git::worktree_add_detached(&repo, &reviewer, "main")
18989 .await
18990 .unwrap();
18991 git::acquire_worktree_config(&repo).await.unwrap();
18992 state.enabled_worktree_config = true;
18993 state.status = RunStatus::Ready;
18994 state.candidates.push(Candidate {
18995 index: 0,
18996 label: 'A',
18997 agent: "implement".into(),
18998 branch: branch.clone(),
18999 worktree: candidate.clone(),
19000 summary: String::new(),
19001 stat: String::new(),
19002 files: 0,
19003 commits: 0,
19004 empty: false,
19005 failed: None,
19006 verified_noop: None,
19007 duration_ms: 0,
19008 folded: false,
19009 });
19010 state.save_under(&home).unwrap();
19011 let config_before = std::fs::read(common.join("config")).unwrap();
19012 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19013 .await
19014 .unwrap();
19015 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19018 assert_eq!(
19019 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19020 0
19021 );
19022 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19023 assert!(
19024 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19025 .await
19026 .is_err()
19027 );
19028 assert!(candidate.exists());
19029 assert!(reviewer.exists());
19030 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19031 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19032 assert_eq!(
19033 git::git(&repo, &["worktree", "list", "--porcelain"])
19034 .await
19035 .unwrap(),
19036 worktrees_before
19037 );
19038 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19039 .await
19040 .unwrap()
19041 .unwrap_err();
19042 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19043 drop(fold);
19044 assert!(candidate.exists() && reviewer.exists());
19045 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19046 assert_eq!(
19047 state.resource_block.as_ref().unwrap().kind,
19048 "repository coordination"
19049 );
19050 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19051 drop(external);
19052 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19053 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19054 .await
19055 .unwrap()
19056 .unwrap();
19057 drop(fold);
19058 assert!(removed.contains(&branch));
19059 assert!(!root.exists());
19060 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19061 assert!(state.candidates[0].folded);
19062 assert!(!state.enabled_worktree_config);
19063 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19064 let enabled = git::git_raw(
19065 &repo,
19066 &["config", "--local", "--get", "extensions.worktreeConfig"],
19067 )
19068 .await
19069 .unwrap();
19070 assert!(!enabled.ok(), "the last config reference was released");
19071 }
19072
19073 #[tokio::test]
19076 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19077 let tmp = tempfile::tempdir().expect("tempdir");
19078 let repo = tmp.path().join("repo");
19079 std::fs::create_dir_all(&repo).unwrap();
19080 init_repo(&repo);
19081 let home = tmp.path().join("home");
19082
19083 let mut config = Config::default();
19084 config.graph.worktree_root = Some(tmp.path().join("wt"));
19085 let mut state = RunState::new(
19086 repo.clone(),
19087 "main".to_owned(),
19088 "deadbeef".to_owned(),
19089 "task".to_owned(),
19090 config,
19091 );
19092 git::git(&repo, &["branch", "magi/x/A", "main"])
19094 .await
19095 .expect("branch");
19096 state.candidates = vec![Candidate {
19097 index: 0,
19098 label: 'A',
19099 agent: "alpha".to_owned(),
19100 branch: "magi/x/A".to_owned(),
19101 worktree: state.worktree_root().join("cand-A"),
19102 summary: String::new(),
19103 stat: String::new(),
19104 files: 0,
19105 commits: 0,
19106 empty: false,
19107 failed: None,
19108 verified_noop: None,
19109 duration_ms: 0,
19110 folded: true,
19111 }];
19112 state.released_to = Some("20260901-000000-new1".to_owned());
19113 state.released_branches = vec!["magi/x/A".to_owned()];
19114
19115 fold_run(&mut state, true, &home).await.expect("fold_run");
19116 state.status = RunStatus::Merged;
19117 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19118 state.merge = Some(crate::run::MergeOutcome {
19119 mode: MergeMode::Pr,
19120 ok: true,
19121 detail: "confirmed merged".into(),
19122 empty: false,
19123 });
19124 fold_run(&mut state, true, &home)
19125 .await
19126 .expect("merged fold");
19127
19128 assert!(
19129 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19130 "the handed-over branch survives a fold"
19131 );
19132 }
19133
19134 #[tokio::test]
19138 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19139 let tmp = tempfile::tempdir().expect("tempdir");
19140 let repo = tmp.path().join("repo");
19141 std::fs::create_dir_all(&repo).unwrap();
19142 init_repo(&repo);
19143 let run = |args: &[&str]| {
19144 let out = std::process::Command::new("git")
19145 .quiet()
19146 .args(args)
19147 .current_dir(&repo)
19148 .output()
19149 .expect("spawn git");
19150 assert!(out.status.success(), "git {args:?}");
19151 };
19152 run(&["branch", "magi/x/A"]);
19153 run(&["checkout", "-q", "-b", "magi/x/B"]);
19154 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19155 run(&["add", "-A"]);
19156 run(&["commit", "-q", "-m", "work"]);
19157 run(&["checkout", "-q", "main"]);
19158
19159 let probe = RunState::new(
19163 repo.clone(),
19164 "main".to_owned(),
19165 "deadbeef".to_owned(),
19166 "task".to_owned(),
19167 Config::default(),
19168 );
19169 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19170 let bare = tmp.path().join("origin.git");
19171 let out = std::process::Command::new("git")
19172 .quiet()
19173 .args(["init", "-q", "--bare"])
19174 .arg(&bare)
19175 .output()
19176 .expect("spawn git");
19177 assert!(out.status.success(), "git init --bare");
19178 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19179 run(&["push", "-q", "origin", "main"]);
19180
19181 let mut state = RunState::new(
19182 repo.clone(),
19183 "main".to_owned(),
19184 "deadbeef".to_owned(),
19185 "task".to_owned(),
19186 Config::default(),
19187 );
19188 state.seeds = vec![refs::Seed {
19189 token: "magi/27b2/A".to_owned(),
19190 kind: refs::SeedKind::Unresolved,
19191 sha: String::new(),
19192 branch: true,
19193 detail: "no branch or commit named magi/27b2/A".to_owned(),
19194 }];
19195
19196 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19197 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19198 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19199 let detail = empty_candidate_detail(&state, "main");
19200 assert!(detail.starts_with("empty candidate"), "{detail}");
19201 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19202 assert!(detail.contains("--start-from"), "{detail}");
19203 }
19204
19205 #[tokio::test]
19214 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19215 let tmp = tempfile::tempdir().expect("tempdir");
19216 let repo = tmp.path().join("repo");
19217 std::fs::create_dir_all(&repo).unwrap();
19218 init_repo(&repo);
19219
19220 let mut config = Config::default();
19221 config.merge.mode = MergeMode::Local;
19222
19223 let mut state = RunState::new(
19224 repo.clone(),
19225 "main".to_owned(),
19226 "deadbeef".to_owned(),
19227 "task".to_owned(),
19228 config,
19229 );
19230 state.candidates = vec![Candidate {
19231 index: 0,
19232 label: 'A',
19233 agent: "alpha".to_owned(),
19234 branch: "does-not-exist".to_owned(),
19235 worktree: repo.clone(),
19236 summary: String::new(),
19237 stat: String::new(),
19238 files: 0,
19239 commits: 0,
19240 empty: false,
19241 failed: None,
19242 verified_noop: None,
19243 duration_ms: 0,
19244 folded: false,
19245 }];
19246 state.tally = Some(Tally {
19247 first_choice: BTreeMap::from([('A', 1)]),
19248 borda: BTreeMap::new(),
19249 winner: 'A',
19250 rankings: 1,
19251 unanimous_initial: true,
19252 deliberated: false,
19253 changed_votes: 0,
19254 unanimous_final: true,
19255 tie_break: None,
19256 judges: 0,
19257 present: 0,
19258 quorum: 0,
19259 met_quorum: true,
19260 uncontested: Some("only candidate A produced a change".to_owned()),
19261 });
19262 state.reviews = vec![ReviewRound {
19263 round: 1,
19264 head: "deadbeef".to_owned(),
19265 verified_head: None,
19266 verified_at: None,
19267 reviews: Vec::new(),
19268 e2e: Vec::new(),
19269 fix: None,
19270 blocking: 0,
19271 answered: 0,
19272 expected: 0,
19273 clean: true,
19274 verify_retried: false,
19275 e2e_deferred: false,
19276 e2e_defer_reason: None,
19277 progressed: false,
19278 vote_split: false,
19279 reconsideration: Vec::new(),
19280 verdict: None,
19281 }];
19282 state.gate = vec![CommandOutcome {
19283 command: "test".to_owned(),
19284 code: Some(0),
19285 output_tail: String::new(),
19286 duration_ms: 0,
19287 resource_blocked: false,
19288 }];
19289 state.gate_ran = true;
19290 state.status = RunStatus::Ready;
19295 state.merge = Some(MergeOutcome {
19296 mode: MergeMode::Local,
19297 ok: false,
19298 detail: "already concluded".to_owned(),
19299 empty: false,
19300 });
19301
19302 let mut runner = Runner {
19303 state,
19304 roles: ResolvedRoles {
19305 implementers: Vec::new(),
19306 judges: Vec::new(),
19307 reviewers: Vec::new(),
19308 fixer: None,
19309 conductor: conductor(),
19310 implementer_roster: Vec::new(),
19311 judge_roster: Vec::new(),
19312 reviewer_roster: Vec::new(),
19313 },
19314 sem: Arc::new(Semaphore::new(1)),
19315 pause: Pause::new(),
19316 interrupt: Pause::new(),
19317 };
19318
19319 runner.merge().await.expect("merge");
19320
19321 assert_eq!(
19322 runner.state.status,
19323 RunStatus::Ready,
19324 "a concluded run's status must not change on reentry"
19325 );
19326 assert_eq!(
19327 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19328 Some("already concluded"),
19329 "merge must not run again once the node already recorded an outcome"
19330 );
19331 }
19332
19333 #[tokio::test]
19342 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19343 let tmp = tempfile::tempdir().expect("tempdir");
19344 let repo = tmp.path().join("repo");
19345 std::fs::create_dir_all(&repo).unwrap();
19346 init_repo(&repo);
19347
19348 let mut config = Config::default();
19349 config.merge.mode = MergeMode::Local;
19350
19351 let mut state = RunState::new(
19352 repo.clone(),
19353 "main".to_owned(),
19354 "deadbeef".to_owned(),
19355 "task".to_owned(),
19356 config,
19357 );
19358 state.candidates = vec![Candidate {
19359 index: 0,
19360 label: 'A',
19361 agent: "alpha".to_owned(),
19362 branch: "does-not-exist".to_owned(),
19363 worktree: repo.clone(),
19364 summary: String::new(),
19365 stat: String::new(),
19366 files: 0,
19367 commits: 0,
19368 empty: false,
19369 failed: None,
19370 verified_noop: None,
19371 duration_ms: 0,
19372 folded: false,
19373 }];
19374 state.tally = Some(Tally {
19375 first_choice: BTreeMap::from([('A', 1)]),
19376 borda: BTreeMap::new(),
19377 winner: 'A',
19378 rankings: 1,
19379 unanimous_initial: true,
19380 deliberated: false,
19381 changed_votes: 0,
19382 unanimous_final: true,
19383 tie_break: None,
19384 judges: 0,
19385 present: 0,
19386 quorum: 0,
19387 met_quorum: true,
19388 uncontested: Some("only candidate A produced a change".to_owned()),
19389 });
19390 state.reviews = vec![ReviewRound {
19391 round: 1,
19392 head: "deadbeef".to_owned(),
19393 verified_head: None,
19394 verified_at: None,
19395 reviews: Vec::new(),
19396 e2e: Vec::new(),
19397 fix: None,
19398 blocking: 0,
19399 answered: 0,
19400 expected: 0,
19401 clean: true,
19402 verify_retried: false,
19403 e2e_deferred: false,
19404 e2e_defer_reason: None,
19405 progressed: false,
19406 vote_split: false,
19407 reconsideration: Vec::new(),
19408 verdict: None,
19409 }];
19410 state.gate = Vec::new();
19412 state.gate_ran = false;
19413 state.status = RunStatus::Gating;
19414
19415 let mut runner = Runner {
19416 state,
19417 roles: ResolvedRoles {
19418 implementers: Vec::new(),
19419 judges: Vec::new(),
19420 reviewers: Vec::new(),
19421 fixer: None,
19422 conductor: conductor(),
19423 implementer_roster: Vec::new(),
19424 judge_roster: Vec::new(),
19425 reviewer_roster: Vec::new(),
19426 },
19427 sem: Arc::new(Semaphore::new(1)),
19428 pause: Pause::new(),
19429 interrupt: Pause::new(),
19430 };
19431
19432 runner.merge().await.expect("merge");
19433
19434 assert!(
19435 runner.state.merge.is_none(),
19436 "an empty gate must never be read as a passing one: {:?}",
19437 runner.state.merge
19438 );
19439 }
19440
19441 #[tokio::test]
19448 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19449 let tmp = tempfile::tempdir().expect("tempdir");
19450 let repo = tmp.path().join("repo");
19451 std::fs::create_dir_all(&repo).unwrap();
19452 init_repo(&repo);
19453
19454 let config = Config::default();
19456
19457 let mut state = RunState::new(
19458 repo.clone(),
19459 "main".to_owned(),
19460 "deadbeef".to_owned(),
19461 "task".to_owned(),
19462 config,
19463 );
19464 state.candidates = vec![Candidate {
19465 index: 0,
19466 label: 'A',
19467 agent: "alpha".to_owned(),
19468 branch: "does-not-exist".to_owned(),
19469 worktree: repo.clone(),
19470 summary: String::new(),
19471 stat: String::new(),
19472 files: 0,
19473 commits: 0,
19474 empty: false,
19475 failed: None,
19476 verified_noop: None,
19477 duration_ms: 0,
19478 folded: false,
19479 }];
19480 state.tally = Some(Tally {
19481 first_choice: BTreeMap::from([('A', 1)]),
19482 borda: BTreeMap::new(),
19483 winner: 'A',
19484 rankings: 1,
19485 unanimous_initial: true,
19486 deliberated: false,
19487 changed_votes: 0,
19488 unanimous_final: true,
19489 tie_break: None,
19490 judges: 0,
19491 present: 0,
19492 quorum: 0,
19493 met_quorum: true,
19494 uncontested: Some("only candidate A produced a change".to_owned()),
19495 });
19496 state.reviews = vec![ReviewRound {
19497 round: 1,
19498 head: "deadbeef".to_owned(),
19499 verified_head: None,
19500 verified_at: None,
19501 reviews: Vec::new(),
19502 e2e: Vec::new(),
19503 fix: None,
19504 blocking: 0,
19505 answered: 0,
19506 expected: 0,
19507 clean: true,
19508 verify_retried: false,
19509 e2e_deferred: false,
19510 e2e_defer_reason: None,
19511 progressed: false,
19512 vote_split: false,
19513 reconsideration: Vec::new(),
19514 verdict: None,
19515 }];
19516
19517 let mut runner = Runner {
19518 state,
19519 roles: ResolvedRoles {
19520 implementers: Vec::new(),
19521 judges: Vec::new(),
19522 reviewers: Vec::new(),
19523 fixer: None,
19524 conductor: conductor(),
19525 implementer_roster: Vec::new(),
19526 judge_roster: Vec::new(),
19527 reviewer_roster: Vec::new(),
19528 },
19529 sem: Arc::new(Semaphore::new(1)),
19530 pause: Pause::new(),
19531 interrupt: Pause::new(),
19532 };
19533
19534 runner.gate().await.expect("gate");
19535 assert!(
19536 runner.state.gate_ran,
19537 "zero configured commands is still a real attempt, not an unrun gate"
19538 );
19539 assert!(runner.state.gate.is_empty());
19540 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19541 assert_ne!(
19542 runner.state.status,
19543 RunStatus::Blocked,
19544 "a gate with nothing to check must not read as failed"
19545 );
19546
19547 runner.merge().await.expect("merge");
19548 assert_eq!(
19549 runner.state.status,
19550 RunStatus::Ready,
19551 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19552 );
19553 }
19554
19555 #[tokio::test]
19566 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19567 crate::run::pin_test_home();
19568 let home = crate::run::home();
19569
19570 let tmp = tempfile::tempdir().expect("tempdir");
19571 let repo = tmp.path().join("repo");
19572 std::fs::create_dir_all(&repo).unwrap();
19573 init_repo(&repo);
19574 let cache_dir = tmp.path().join("target");
19577
19578 let mut config = Config::default();
19579 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19580 config.graph.timeout_verify = Some(2);
19584 config.graph.timeout_resource_wait = 0;
19585
19586 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19587 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19588 .expect("no io error acquiring directly")
19589 {
19590 crate::cache::AcquireOutcome::Acquired(g) => g,
19591 crate::cache::AcquireOutcome::Busy(b) => {
19592 panic!("expected the direct acquire to win the lease first: {b:?}")
19593 }
19594 };
19595
19596 let mut state = RunState::new(
19597 repo.clone(),
19598 "main".to_owned(),
19599 "deadbeef".to_owned(),
19600 "task".to_owned(),
19601 config,
19602 );
19603 state.candidates = vec![Candidate {
19604 index: 0,
19605 label: 'A',
19606 agent: "alpha".to_owned(),
19607 branch: "does-not-exist".to_owned(),
19608 worktree: repo.clone(),
19609 summary: String::new(),
19610 stat: String::new(),
19611 files: 0,
19612 commits: 0,
19613 empty: false,
19614 failed: None,
19615 verified_noop: None,
19616 duration_ms: 0,
19617 folded: false,
19618 }];
19619 state.tally = Some(Tally {
19620 first_choice: BTreeMap::from([('A', 1)]),
19621 borda: BTreeMap::new(),
19622 winner: 'A',
19623 rankings: 1,
19624 unanimous_initial: true,
19625 deliberated: false,
19626 changed_votes: 0,
19627 unanimous_final: true,
19628 tie_break: None,
19629 judges: 0,
19630 present: 0,
19631 quorum: 0,
19632 met_quorum: true,
19633 uncontested: Some("only candidate A produced a change".to_owned()),
19634 });
19635 state.reviews = vec![ReviewRound {
19636 round: 1,
19637 head: "deadbeef".to_owned(),
19638 verified_head: None,
19639 verified_at: None,
19640 reviews: Vec::new(),
19641 e2e: Vec::new(),
19642 fix: None,
19643 blocking: 0,
19644 answered: 0,
19645 expected: 0,
19646 clean: true,
19647 verify_retried: false,
19648 e2e_deferred: false,
19649 e2e_defer_reason: None,
19650 progressed: false,
19651 vote_split: false,
19652 reconsideration: Vec::new(),
19653 verdict: None,
19654 }];
19655
19656 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19657 state.reviews[0].e2e = vec![CommandOutcome {
19658 command: state.config.verify.e2e[0].clone(),
19659 code: Some(0),
19660 output_tail: String::new(),
19661 duration_ms: 0,
19662 resource_blocked: false,
19663 }];
19664 let mut runner = Runner {
19665 state,
19666 roles: ResolvedRoles {
19667 implementers: Vec::new(),
19668 judges: Vec::new(),
19669 reviewers: Vec::new(),
19670 fixer: None,
19671 conductor: conductor(),
19672 implementer_roster: Vec::new(),
19673 judge_roster: Vec::new(),
19674 reviewer_roster: Vec::new(),
19675 },
19676 sem: Arc::new(Semaphore::new(1)),
19677 pause: Pause::new(),
19678 interrupt: Pause::new(),
19679 };
19680
19681 runner.gate().await.expect("gate");
19682 assert!(runner.state.resource_block.is_none());
19683 assert!(
19684 !runner
19685 .state
19686 .events
19687 .iter()
19688 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19689 "a gate with nothing to run must never request the held cache lease"
19690 );
19691 assert!(
19692 runner.state.gate_ran,
19693 "zero commands is still a real, immediate attempt"
19694 );
19695 assert!(runner.state.gate.is_empty());
19696 assert_ne!(
19697 runner.state.status,
19698 RunStatus::Blocked,
19699 "must not read as resource-blocked on a lease it never asked for"
19700 );
19701 }
19702
19703 #[cfg(unix)]
19706 #[tokio::test]
19707 async fn verification_is_standalone_inside_a_live_worker() {
19708 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19709 if std::env::var_os(CHILD).is_none() {
19710 let home = tempfile::tempdir().unwrap();
19711 let generation = crate::worker::pin_current(home.path()).unwrap();
19712 let output = std::process::Command::new(std::env::current_exe().unwrap())
19713 .args([
19714 "--exact",
19715 "graph::tests::verification_is_standalone_inside_a_live_worker",
19716 "--nocapture",
19717 ])
19718 .env(CHILD, "1")
19719 .env("MAGI_HOME", home.path())
19720 .env("MAGI_WORKER_HOME", home.path())
19721 .env("MAGI_WORKER_ID", "abcdef")
19722 .env("MAGI_WORKER_GENERATION", &generation.id)
19723 .env("MAGI_WORKER_EXE", &generation.executable)
19724 .env("MAGI_RUN", "verification-parent")
19725 .env("MAGI_NODE", "gate")
19726 .env("MAGI_SEAT", "worker-fixture")
19727 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19728 .env("MAGI_NO_AUTOUPDATE", "0")
19729 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19730 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19731 .env(
19732 "VERIFY_EXPECT_TOOLCHAIN",
19733 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19734 )
19735 .env(
19736 "VERIFY_EXPECT_CACHE",
19737 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19738 )
19739 .output()
19740 .unwrap();
19741 assert!(
19742 output.status.success()
19743 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19744 "{}\n{}",
19745 String::from_utf8_lossy(&output.stdout),
19746 String::from_utf8_lossy(&output.stderr)
19747 );
19748 return;
19749 }
19750 let home = crate::run::home();
19751 let directory = home.join("workers/jobs/abcdef");
19752 std::fs::create_dir_all(&directory).unwrap();
19753 let job = crate::worker::Job {
19754 id: "abcdef".into(),
19755 sequence: 1,
19756 kind: "task".into(),
19757 key: "task:verification".into(),
19758 payload: serde_json::json!({}),
19759 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19760 version: Some(env!("CARGO_PKG_VERSION").into()),
19761 status: crate::worker::Status::Running,
19762 result: None,
19763 pid: Some(std::process::id()),
19764 created_at: 0,
19765 finished_at: None,
19766 };
19767 std::fs::write(
19768 directory.join("job.json"),
19769 serde_json::to_vec(&job).unwrap(),
19770 )
19771 .unwrap();
19772 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19773 let assert_worker = || {
19774 assert_eq!(
19775 crate::worker::current_job(&home).unwrap().unwrap().id,
19776 job.id
19777 );
19778 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19779 assert_eq!(
19780 crate::worker::helper_executable(&home).unwrap(),
19781 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19782 );
19783 assert!(
19784 crate::updater::Checker::new(&crate::config::Update {
19785 mode: crate::config::UpdateMode::Notify,
19786 interval: None,
19787 auto_restart: false,
19788 })
19789 .is_none(),
19790 "the driver remains pinned, even while tests are standalone"
19791 );
19792 };
19793 assert_worker();
19794 let repo = home.join("repo");
19795 std::fs::create_dir_all(&repo).unwrap();
19796 init_repo(&repo);
19797 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19800 test -z "${MAGI_WORKER_GENERATION+x}" &&
19801 test -z "${MAGI_WORKER_HOME+x}" &&
19802 test -z "${MAGI_WORKER_EXE+x}" &&
19803 test "$MAGI_HOME/repo" -ef . &&
19804 test "$MAGI_RUN" = verification-parent &&
19805 test "$MAGI_NODE" = gate &&
19806 test "$MAGI_SEAT" = worker-fixture &&
19807 test -n "$MAGI_PROCESS_OWNER" &&
19808 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19809 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19810 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19811 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19812 assert!(
19814 !std::process::Command::new("sh")
19815 .args(["-c", probe])
19816 .current_dir(&repo)
19817 .status()
19818 .unwrap()
19819 .success()
19820 );
19821 let unrelated_home = tempfile::tempdir().unwrap();
19822 assert!(
19823 crate::worker::current_job(unrelated_home.path()).is_err(),
19824 "an inherited worker id cannot be resolved in a unit fixture's home"
19825 );
19826 let mut runner = runner_at(RunStatus::Gating);
19827 runner.state.repo = repo.clone();
19828 runner.state.config.disk.min_free_bytes = 0;
19829 runner.state.config.coordination = crate::config::test_coordination();
19830 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19831 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19832 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19833 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19834 let winner = Candidate {
19835 index: 0,
19836 label: 'A',
19837 agent: "fixture".into(),
19838 branch: "main".into(),
19839 worktree: repo.clone(),
19840 summary: String::new(),
19841 stat: String::new(),
19842 files: 0,
19843 commits: 0,
19844 empty: false,
19845 failed: None,
19846 verified_noop: None,
19847 duration_ms: 0,
19848 folded: false,
19849 };
19850 let (e2e, retried, pids) = run_e2e_with_retry(
19851 &mut runner.state,
19852 &["sh".into(), "-c".into()],
19853 &[format!("{probe} && printf initial")],
19854 &repo,
19855 Duration::from_secs(10),
19856 "initial",
19857 )
19858 .await;
19859 assert!(!retried && pids.is_empty());
19860 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
19861 assert_eq!(e2e[0].output_tail, "initial");
19862 let gate = runner.run_gate(&winner).await.unwrap();
19863 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
19864 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
19867 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
19868 passed: false,
19869 outcomes: Vec::new(),
19870 });
19871 runner.state.save().unwrap();
19872 let mut resumed = RunState::load(&runner.state.id).unwrap();
19873 let wait = crate::repo_lock::WaitPolicy::default();
19874 assert!(matches!(
19875 check_gate_repair_e2e(&mut resumed, &winner, &wait)
19876 .await
19877 .unwrap(),
19878 GateFix::Retry
19879 ));
19880 let saved = RunState::load(&resumed.id).unwrap();
19881 let repair = saved.gate_repair_e2e.unwrap();
19882 assert!(repair.passed);
19883 assert_eq!(repair.outcomes.len(), 1);
19884 assert!(repair.outcomes[0].ok());
19885 let landing = run_landing_gate(&mut resumed, &winner, &wait)
19886 .await
19887 .unwrap();
19888 assert_eq!(landing.len(), 1);
19889 assert!(landing[0].ok(), "{landing:?}");
19890 assert_eq!(
19891 std::fs::read_to_string(repo.join("phases")).unwrap(),
19892 "pre\ngate\ne2e\npre\ngate\n"
19893 );
19894 assert_worker();
19895 }
19896
19897 #[cfg(unix)]
19898 #[tokio::test]
19899 async fn verification_keeps_the_panic_and_the_full_output() {
19900 crate::run::pin_test_home();
19901 let tmp = tempfile::tempdir().unwrap();
19902 let mut config = Config::default();
19903 config.disk.min_free_bytes = 0;
19904 let mut state = RunState::new(
19905 tmp.path().to_owned(),
19906 "main".into(),
19907 "base".into(),
19908 "verification evidence".into(),
19909 config,
19910 );
19911 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
19912 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
19913 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
19914 let mut names = Vec::new();
19915 for _ in 0..2 {
19916 let (outcomes, _) = run_commands(
19917 &mut state,
19918 "gate",
19919 "gate",
19920 0,
19921 &["sh".into(), "-c".into()],
19922 &[script.into()],
19923 tmp.path(),
19924 Duration::from_secs(60),
19925 )
19926 .await;
19927 let tail = &outcomes[0].output_tail;
19928 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
19929 assert!(tail.contains("boom assertion"));
19930 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
19931 let line = tail.lines().next().unwrap();
19932 names.push(line.to_owned());
19933 }
19934 assert_ne!(names[0], names[1], "the second run must not overwrite");
19935 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
19936 assert!(full.len() > 500_000 && full.contains("boom assertion"));
19937 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
19938
19939 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
19941 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
19942 let (outcomes, _) = run_commands(
19943 &mut state,
19944 "gate",
19945 "gate",
19946 0,
19947 &["sh".into(), "-c".into()],
19948 &["echo failure-output; exit 7".into()],
19949 tmp.path(),
19950 Duration::from_secs(10),
19951 )
19952 .await;
19953 assert_eq!(outcomes[0].code, Some(7));
19954 assert!(state.events.iter().any(|e| {
19955 e.node == "gate" && e.message.contains("could not save the full gate output:")
19956 }));
19957 }
19958
19959 #[cfg(unix)]
19960 #[tokio::test]
19961 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
19962 crate::run::pin_test_home();
19963 let tmp = tempfile::tempdir().unwrap();
19964 let mut config = Config::default();
19965 config.disk.min_free_bytes = 0;
19966 let mut state = RunState::new(
19967 tmp.path().to_owned(),
19968 "main".into(),
19969 "base".into(),
19970 "verification evidence".into(),
19971 config,
19972 );
19973 let (outcomes, pids) = run_commands(
19974 &mut state,
19975 "gate",
19976 "gate",
19977 1,
19978 &["sh".into(), "-c".into()],
19979 &[
19980 "printf before-timeout; exec sleep 30".into(),
19981 "kill -TERM $$".into(),
19982 "exit 7".into(),
19983 "printf success".into(),
19984 ],
19985 tmp.path(),
19986 Duration::from_secs(1),
19987 )
19988 .await;
19989 wait_for_timed_out_children_to_die(&pids).await;
19990 assert_eq!(
19991 pids.len(),
19992 1,
19993 "unleased timeout retains its death-check PID"
19994 );
19995 assert_eq!(outcomes.len(), 4);
19996 assert_eq!(outcomes[0].code, None);
19997 assert!(outcomes[0].output_tail.contains("before-timeout"));
19998 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
19999 assert_eq!(outcomes[1].code, None);
20000 assert!(outcomes[1].output_tail.contains("command terminated:"));
20001 assert!(!outcomes[1].output_tail.contains("timed out"));
20002 assert_eq!(outcomes[2].code, Some(7));
20003 assert!(!outcomes[2].output_tail.contains("timed out"));
20004 assert_eq!(outcomes[3].code, Some(0));
20005 assert_eq!(outcomes[3].output_tail, "success");
20006 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20007 }
20008
20009 #[tokio::test]
20019 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20020 crate::run::pin_test_home();
20021
20022 let tmp = tempfile::tempdir().expect("tempdir");
20023 let repo = tmp.path().join("repo");
20024 std::fs::create_dir_all(&repo).unwrap();
20025 init_repo(&repo);
20026
20027 let mut config = Config::default();
20028 config.disk.min_free_bytes = 0;
20029 config.verify.gate = vec![
20030 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20031 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20032 .to_owned(),
20033 ];
20034
20035 let mut state = RunState::new(
20036 repo.clone(),
20037 "main".to_owned(),
20038 "deadbeef".to_owned(),
20039 "task".to_owned(),
20040 config,
20041 );
20042 let run_id = state.id.clone();
20043 state.candidates = vec![Candidate {
20044 index: 0,
20045 label: 'A',
20046 agent: "alpha".to_owned(),
20047 branch: "does-not-exist".to_owned(),
20048 worktree: repo.clone(),
20049 summary: String::new(),
20050 stat: String::new(),
20051 files: 0,
20052 commits: 0,
20053 empty: false,
20054 failed: None,
20055 verified_noop: None,
20056 duration_ms: 0,
20057 folded: false,
20058 }];
20059 state.tally = Some(Tally {
20060 first_choice: BTreeMap::from([('A', 1)]),
20061 borda: BTreeMap::new(),
20062 winner: 'A',
20063 rankings: 1,
20064 unanimous_initial: true,
20065 deliberated: false,
20066 changed_votes: 0,
20067 unanimous_final: true,
20068 tie_break: None,
20069 judges: 0,
20070 present: 0,
20071 quorum: 0,
20072 met_quorum: true,
20073 uncontested: Some("only candidate A produced a change".to_owned()),
20074 });
20075 state.reviews = vec![ReviewRound {
20076 round: 1,
20077 head: "deadbeef".to_owned(),
20078 verified_head: None,
20079 verified_at: None,
20080 reviews: Vec::new(),
20081 e2e: Vec::new(),
20082 fix: None,
20083 blocking: 0,
20084 answered: 0,
20085 expected: 0,
20086 clean: true,
20087 verify_retried: false,
20088 e2e_deferred: false,
20089 e2e_defer_reason: None,
20090 progressed: false,
20091 vote_split: false,
20092 reconsideration: Vec::new(),
20093 verdict: None,
20094 }];
20095
20096 let mut runner = Runner {
20097 state,
20098 roles: ResolvedRoles {
20099 implementers: Vec::new(),
20100 judges: Vec::new(),
20101 reviewers: Vec::new(),
20102 fixer: None,
20103 conductor: conductor(),
20104 implementer_roster: Vec::new(),
20105 judge_roster: Vec::new(),
20106 reviewer_roster: Vec::new(),
20107 },
20108 sem: Arc::new(Semaphore::new(1)),
20109 pause: Pause::new(),
20110 interrupt: Pause::new(),
20111 };
20112
20113 let started_marker = repo.join("started.marker");
20114 let release_marker = repo.join("release.marker");
20115 let poller = tokio::spawn(async move {
20116 for _ in 0..100 {
20121 if started_marker.exists()
20122 && let Ok(s) = crate::run::RunState::load(&run_id)
20123 && let Some(a) = s.active.get("gate")
20124 {
20125 std::fs::write(&release_marker, b"go").expect("release marker");
20126 return Some(a.clone());
20127 }
20128 tokio::time::sleep(Duration::from_millis(50)).await;
20129 }
20130 None
20131 });
20132
20133 runner.gate().await.expect("gate");
20134 let captured = poller.await.expect("poller task");
20135 let captured = captured.expect(
20136 "the poller never saw a `gate` task entry in run.json while the command was \
20137 still blocked on its own release marker",
20138 );
20139
20140 assert_eq!(captured.task.as_deref(), Some("gate"));
20141 assert_eq!(captured.node, "gate");
20142 assert_eq!(captured.index, Some(1));
20143 assert_eq!(captured.total, Some(1));
20144 assert!(
20145 captured
20146 .command
20147 .as_deref()
20148 .is_some_and(|c| c.contains("started.marker")),
20149 "{captured:?}"
20150 );
20151
20152 assert!(
20153 runner.state.active.is_empty(),
20154 "the entry must be cleared once the command actually finished: {:?}",
20155 runner.state.active
20156 );
20157 assert!(runner.state.gate_ran);
20158 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20159 }
20160
20161 #[tokio::test]
20164 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20165 use crate::verdict::{Finding, ReviewVote, Severity};
20166 crate::run::pin_test_home();
20167 let tmp = tempfile::tempdir().expect("tempdir");
20168 let repo = tmp.path().join("repo");
20169 std::fs::create_dir_all(&repo).unwrap();
20170 init_repo(&repo);
20171
20172 for (severity, vote, expect) in [
20173 (Severity::Major, ReviewVote::Reject, true),
20174 (Severity::Minor, ReviewVote::Reject, false),
20175 (Severity::Major, ReviewVote::Approve, false),
20176 ] {
20177 let mut round = review_round(false, 1, 1, 1, false, true);
20178 round.reviews = vec![ReviewRecord {
20179 reviewer: 1,
20180 agent: "alpha".to_owned(),
20181 summary: String::new(),
20182 findings: vec![Finding {
20183 id: "R1-1-1".to_owned(),
20184 severity,
20185 file: None,
20186 line: None,
20187 title: "t".to_owned(),
20188 detail: String::new(),
20189 }],
20190 vote: Some(vote),
20191 failed: None,
20192 duration_ms: 0,
20193 attempts: 0,
20194 }];
20195 let mut state = RunState::new(
20196 repo.clone(),
20197 "main".to_owned(),
20198 "deadbeef".to_owned(),
20199 "task".to_owned(),
20200 Config::default(),
20201 );
20202 state.reviews = vec![round];
20203 let mut runner = Runner {
20204 state,
20205 roles: ResolvedRoles {
20206 implementers: Vec::new(),
20207 judges: Vec::new(),
20208 reviewers: Vec::new(),
20209 fixer: None,
20210 conductor: conductor(),
20211 implementer_roster: Vec::new(),
20212 judge_roster: Vec::new(),
20213 reviewer_roster: Vec::new(),
20214 },
20215 sem: Arc::new(Semaphore::new(1)),
20216 pause: Pause::new(),
20217 interrupt: Pause::new(),
20218 };
20219 let shell = runner.state.config.shell();
20220 runner
20221 .stop_reviewing("round budget spent", &shell, &repo)
20222 .await
20223 .expect("stop_reviewing");
20224 assert_eq!(runner.state.status, RunStatus::Gating);
20225 assert_eq!(
20226 runner.state.contested_handoff.is_some(),
20227 expect,
20228 "{severity:?} + {vote:?}"
20229 );
20230 }
20231 }
20232
20233 #[tokio::test]
20246 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20247 crate::run::pin_test_home();
20248 let home = crate::run::home();
20249
20250 let tmp = tempfile::tempdir().expect("tempdir");
20251 let repo = tmp.path().join("repo");
20252 std::fs::create_dir_all(&repo).unwrap();
20253 init_repo(&repo);
20254 let head = crate::git::rev_parse(&repo, "HEAD")
20255 .await
20256 .expect("rev-parse");
20257 let cache_dir = tmp.path().join("target");
20260
20261 let mut config = Config::default();
20262 config.verify.e2e = vec![format!(
20263 "CARGO_TARGET_DIR='{}' test -f README.md",
20264 cache_dir.display()
20265 )];
20266 config.graph.review_rounds = 1;
20267 config.graph.timeout_verify = Some(2);
20270 config.graph.timeout_resource_wait = 2;
20271
20272 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20273 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20274 .expect("no io error acquiring directly")
20275 {
20276 crate::cache::AcquireOutcome::Acquired(g) => g,
20277 crate::cache::AcquireOutcome::Busy(b) => {
20278 panic!("expected the direct acquire to win the lease first: {b:?}")
20279 }
20280 };
20281
20282 let mut state = RunState::new(
20283 repo.clone(),
20284 "main".to_owned(),
20285 head.clone(),
20286 "task".to_owned(),
20287 config,
20288 );
20289 state.candidates = vec![Candidate {
20290 index: 0,
20291 label: 'A',
20292 agent: "alpha".to_owned(),
20293 branch: "does-not-exist".to_owned(),
20294 worktree: repo.clone(),
20295 summary: String::new(),
20296 stat: String::new(),
20297 files: 0,
20298 commits: 0,
20299 empty: false,
20300 failed: None,
20301 verified_noop: None,
20302 duration_ms: 0,
20303 folded: false,
20304 }];
20305 state.tally = Some(Tally {
20306 first_choice: BTreeMap::from([('A', 1)]),
20307 borda: BTreeMap::new(),
20308 winner: 'A',
20309 rankings: 1,
20310 unanimous_initial: true,
20311 deliberated: false,
20312 changed_votes: 0,
20313 unanimous_final: true,
20314 tie_break: None,
20315 judges: 0,
20316 present: 0,
20317 quorum: 0,
20318 met_quorum: true,
20319 uncontested: Some("only candidate A produced a change".to_owned()),
20320 });
20321 state.reviews = vec![ReviewRound {
20325 round: 1,
20326 head: head.clone(),
20327 verified_head: None,
20328 verified_at: None,
20329 reviews: Vec::new(),
20330 e2e: Vec::new(),
20331 fix: None,
20332 blocking: 1,
20333 answered: 1,
20334 expected: 1,
20335 clean: false,
20336 verify_retried: false,
20337 e2e_deferred: true,
20338 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20339 progressed: false,
20340 vote_split: false,
20341 reconsideration: Vec::new(),
20342 verdict: None,
20343 }];
20344
20345 let mut runner = Runner {
20346 state,
20347 roles: ResolvedRoles {
20348 implementers: Vec::new(),
20349 judges: Vec::new(),
20350 reviewers: Vec::new(),
20351 fixer: None,
20352 conductor: conductor(),
20353 implementer_roster: Vec::new(),
20354 judge_roster: Vec::new(),
20355 reviewer_roster: Vec::new(),
20356 },
20357 sem: Arc::new(Semaphore::new(1)),
20358 pause: Pause::new(),
20359 interrupt: Pause::new(),
20360 };
20361
20362 let shell = runner.state.config.shell();
20363 runner
20364 .stop_reviewing("round budget spent", &shell, &repo)
20365 .await
20366 .expect("stop_reviewing");
20367
20368 let last = runner.state.reviews.last().expect("round record");
20369 assert_eq!(
20370 last.e2e_status(),
20371 E2eStatus::ResourceBlocked,
20372 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20373 failed: {last:?}"
20374 );
20375 assert_eq!(
20376 last.verified_head.as_deref(),
20377 Some(head.as_str()),
20378 "which commit this attempt targeted is known even though nothing finished checking \
20379 it"
20380 );
20381 let first_attempt_at = last
20382 .verified_at
20383 .expect("when this attempt ran is known too");
20384 assert_ne!(
20385 runner.state.status,
20386 RunStatus::Blocked,
20387 "contention is evidence about the machine, not the patch — it must not settle the \
20388 run as blocked: {:?}",
20389 runner.state.status
20390 );
20391 assert!(
20392 !runner
20393 .state
20394 .events
20395 .iter()
20396 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20397 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20398 runner.state.events
20399 );
20400
20401 runner
20406 .stop_reviewing("round budget spent", &shell, &repo)
20407 .await
20408 .expect("stop_reviewing retry");
20409 assert_eq!(
20410 runner.state.reviews.len(),
20411 1,
20412 "no new round was started: {:?}",
20413 runner.state.reviews
20414 );
20415 let last = runner.state.reviews.last().expect("round record");
20416 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20417 assert!(
20418 last.verified_at.expect("still known") > first_attempt_at,
20419 "a second reentry must be a fresh attempt, not a stale copy of the first"
20420 );
20421 assert_ne!(runner.state.status, RunStatus::Blocked);
20422
20423 held.release();
20424 }
20425
20426 #[cfg(unix)]
20427 #[tokio::test]
20428 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20429 ask_test_home();
20430 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20431 let tmp = tempfile::tempdir().unwrap();
20432 let repo = tmp.path().join("repo");
20433 std::fs::create_dir(&repo).unwrap();
20434 init_repo(&repo);
20435 let run_git = |args: &[&str]| {
20436 let out = std::process::Command::new("git")
20437 .args(args)
20438 .current_dir(&repo)
20439 .output()
20440 .unwrap();
20441 assert!(
20442 out.status.success(),
20443 "{args:?}: {}",
20444 String::from_utf8_lossy(&out.stderr)
20445 );
20446 };
20447 let origin = tmp.path().join("origin.git");
20448 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20449 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20450 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20451 run_git(&["add", "spec.rs"]);
20452 run_git(&["commit", "-m", "initial spec"]);
20453 run_git(&["push", "origin", "main"]);
20454 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20455 run_git(&["checkout", "-b", "fix/review"]);
20456 std::fs::write(
20457 repo.join("check.rs"),
20458 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20459 )
20460 .unwrap();
20461 run_git(&["add", "check.rs"]);
20462 run_git(&["commit", "-m", "use spec"]);
20463 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20464 run_git(&["checkout", "main"]);
20465 std::fs::write(
20466 repo.join("spec.rs"),
20467 "struct AgentSpec { max_turn_secs: u64 }\n",
20468 )
20469 .unwrap();
20470 run_git(&["add", "spec.rs"]);
20471 run_git(&["commit", "-m", "extend spec"]);
20472 run_git(&["push", "origin", "main"]);
20473 run_git(&["checkout", "fix/review"]);
20474
20475 let mut runner = runner_at(RunStatus::Reviewing);
20476 runner.state.repo = repo.clone();
20477 runner.state.base_commit = base;
20478 runner.state.pr_head = Some("fix/review".into());
20479 runner.state.config.graph.review_rounds = max_rounds;
20480 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20481 runner.state.config.graph.sessions = false;
20482 runner.state.config.disk.min_free_bytes = 0;
20483 runner.state.config.verify.e2e =
20484 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20485 runner.state.candidates = vec![Candidate {
20486 index: 0,
20487 label: 'A',
20488 agent: "fixture".into(),
20489 branch: "fix/review".into(),
20490 worktree: repo.clone(),
20491 summary: String::new(),
20492 stat: String::new(),
20493 files: 1,
20494 commits: 1,
20495 empty: false,
20496 failed: None,
20497 verified_noop: None,
20498 duration_ms: 0,
20499 folded: false,
20500 }];
20501 runner.state.tally = Some(Tally {
20502 first_choice: BTreeMap::new(),
20503 borda: BTreeMap::new(),
20504 winner: 'A',
20505 rankings: 0,
20506 unanimous_initial: true,
20507 deliberated: false,
20508 changed_votes: 0,
20509 unanimous_final: true,
20510 tie_break: None,
20511 judges: 0,
20512 present: 0,
20513 quorum: 0,
20514 met_quorum: true,
20515 uncontested: Some("review-only".into()),
20516 });
20517 let mut last = review_round(false, 0, 1, 1, false, false);
20518 last.head = reviewed.clone();
20519 last.e2e_deferred = true;
20520 last.e2e[0].resource_blocked = true;
20521 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20522 if max_rounds == 1 {
20523 last.e2e.clear();
20524 }
20525 runner.state.reviews = vec![last];
20526 let calls = tmp.path().join("fix-calls");
20527 let mut fixer = spec("repair");
20528 fixer
20529 .env
20530 .insert("FIX_CALLS".into(), calls.display().to_string());
20531 fixer.command = vec!["sh".into(), "-c".into(), r#"
20532 echo fix >> "$FIX_CALLS"
20533 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20534 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20535 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20536 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20537 "#.into()];
20538 let mut reviewer = spec("reviewer");
20539 reviewer.command = vec![
20540 "sh".into(),
20541 "-c".into(),
20542 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20543 .into(),
20544 ];
20545 runner.roles.fixer = Some(vec![fixer]);
20546 runner.roles.reviewers = vec![reviewer];
20547
20548 runner.sync_to_base().await.unwrap();
20549 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20550 assert_ne!(rebased, reviewed);
20551 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20552 if reentry {
20553 let shell = runner.state.config.shell();
20556 runner
20557 .stop_unresolved_review("deferred verification", &shell, &repo)
20558 .await
20559 .unwrap();
20560 assert!(runner.catchup_repairable());
20561 assert!(!calls.exists());
20562 runner.state = RunState::load(&runner.state.id).unwrap();
20563 assert!(runner.catchup_repairable());
20564 }
20565 runner.review_loop().await.unwrap();
20566 let first = &runner.state.reviews[0];
20567 assert_eq!(first.head, reviewed);
20568 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20569 assert!(
20570 first.verify_retried,
20571 "a reproduced build failure gets exactly one retry: {:?}",
20572 first.e2e
20573 );
20574 assert!(first.e2e[0].output_tail.contains("E0063"));
20575 assert!(!runner.catchup_repairable());
20576 if max_rounds == 2 {
20577 assert_eq!(
20578 runner.state.status,
20579 RunStatus::Gating,
20580 "{}",
20581 crate::report::run(&runner.state)
20582 );
20583 assert_eq!(runner.state.reviews.len(), 2);
20584 assert!(first.fix.as_ref().unwrap().committed);
20585 assert!(runner.state.reviews[1].clean);
20586 assert_eq!(
20587 runner
20588 .state
20589 .reviews
20590 .iter()
20591 .filter(|r| r.fix.is_some())
20592 .count(),
20593 1
20594 );
20595 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20596 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20597 } else {
20598 assert_eq!(runner.state.status, RunStatus::Blocked);
20599 assert!(first.fix.is_none());
20600 assert!(!calls.exists());
20601 let recovery = crate::recovery::assess(
20602 "prior",
20603 Some(&runner.state),
20604 &runner.landing_base(),
20605 crate::recovery::Relation::Diverged,
20606 );
20607 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20608 let reasons = recovery.reasons.join("\n");
20609 assert!(
20610 reasons.contains(&reviewed) && reasons.contains(&rebased),
20611 "{reasons}"
20612 );
20613 assert!(
20614 reasons.contains("base sync to")
20615 && reasons.contains("additional review is required"),
20616 "{reasons}"
20617 );
20618 }
20619 }
20620 }
20621
20622 #[tokio::test]
20634 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20635 crate::run::pin_test_home();
20636 let home = crate::run::home();
20637
20638 let tmp = tempfile::tempdir().expect("tempdir");
20639 let repo = tmp.path().join("repo");
20640 std::fs::create_dir_all(&repo).unwrap();
20641 init_repo(&repo);
20642 let head = crate::git::rev_parse(&repo, "HEAD")
20643 .await
20644 .expect("rev-parse");
20645 let cache_dir = tmp.path().join("target");
20646
20647 let mut config = Config::default();
20648 config.verify.e2e = vec![format!(
20649 "CARGO_TARGET_DIR='{}' test -f README.md",
20650 cache_dir.display()
20651 )];
20652 config.graph.review_rounds = 1;
20653 config.graph.timeout_verify = Some(2);
20654 config.graph.timeout_resource_wait = 2;
20655
20656 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20657 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20658 .expect("no io error acquiring directly")
20659 {
20660 crate::cache::AcquireOutcome::Acquired(g) => g,
20661 crate::cache::AcquireOutcome::Busy(b) => {
20662 panic!("expected the direct acquire to win the lease first: {b:?}")
20663 }
20664 };
20665
20666 let mut state = RunState::new(
20667 repo.clone(),
20668 "main".to_owned(),
20669 head.clone(),
20670 "task".to_owned(),
20671 config,
20672 );
20673 state.candidates = vec![Candidate {
20674 index: 0,
20675 label: 'A',
20676 agent: "alpha".to_owned(),
20677 branch: "does-not-exist".to_owned(),
20678 worktree: repo.clone(),
20679 summary: String::new(),
20680 stat: String::new(),
20681 files: 0,
20682 commits: 0,
20683 empty: false,
20684 failed: None,
20685 verified_noop: None,
20686 duration_ms: 0,
20687 folded: false,
20688 }];
20689 state.tally = Some(Tally {
20690 first_choice: BTreeMap::from([('A', 1)]),
20691 borda: BTreeMap::new(),
20692 winner: 'A',
20693 rankings: 1,
20694 unanimous_initial: true,
20695 deliberated: false,
20696 changed_votes: 0,
20697 unanimous_final: true,
20698 tie_break: None,
20699 judges: 0,
20700 present: 0,
20701 quorum: 0,
20702 met_quorum: true,
20703 uncontested: Some("only candidate A produced a change".to_owned()),
20704 });
20705 state.reviews = vec![ReviewRound {
20709 round: 1,
20710 head: head.clone(),
20711 verified_head: Some(head.clone()),
20712 verified_at: Some(jiff::Timestamp::now()),
20713 reviews: Vec::new(),
20714 e2e: vec![CommandOutcome {
20715 command: format!(
20716 "CARGO_TARGET_DIR='{}' test -f README.md",
20717 cache_dir.display()
20718 ),
20719 code: None,
20720 output_tail: "waiting for the shared build cache".to_owned(),
20721 duration_ms: 0,
20722 resource_blocked: true,
20723 }],
20724 fix: None,
20725 blocking: 1,
20726 answered: 1,
20727 expected: 1,
20728 clean: false,
20729 verify_retried: false,
20730 e2e_deferred: false,
20731 e2e_defer_reason: None,
20732 progressed: false,
20733 vote_split: false,
20734 reconsideration: Vec::new(),
20735 verdict: None,
20736 }];
20737
20738 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20739 let mut runner = Runner {
20740 state,
20741 roles: ResolvedRoles {
20742 implementers: Vec::new(),
20743 judges: Vec::new(),
20744 reviewers: Vec::new(),
20745 fixer: None,
20746 conductor: conductor(),
20747 implementer_roster: Vec::new(),
20748 judge_roster: Vec::new(),
20749 reviewer_roster: Vec::new(),
20750 },
20751 sem: Arc::new(Semaphore::new(1)),
20752 pause: Pause::new(),
20753 interrupt: Pause::new(),
20754 };
20755
20756 runner.review_loop().await.expect("review_loop");
20761
20762 assert_eq!(
20763 runner.state.reviews.len(),
20764 1,
20765 "no new round was started on top of the unresolved one: {:?}",
20766 runner.state.reviews
20767 );
20768 let last = &runner.state.reviews[0];
20769 assert_eq!(
20770 last.e2e_status(),
20771 E2eStatus::ResourceBlocked,
20772 "still contended: {last:?}"
20773 );
20774 assert!(
20775 last.verified_at.expect("still known") > first_attempt_at,
20776 "review_loop must have actually retried the check, not left it exactly as found"
20777 );
20778 assert_ne!(
20779 runner.state.status,
20780 RunStatus::Blocked,
20781 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20782 runner.state.status
20783 );
20784
20785 held.release();
20786 }
20787
20788 #[tokio::test]
20789 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20790 crate::run::pin_test_home();
20791 for status in [RunStatus::Blocked, RunStatus::Gating] {
20792 let mut runner = runner_at(status);
20793 runner.state.config.graph.land = true;
20794 runner.state.gate_ran = true;
20795 runner.state.merge = Some(MergeOutcome {
20796 mode: MergeMode::Pr,
20797 ok: false,
20798 detail: "the fixer produced no commit".into(),
20799 empty: false,
20800 });
20801 let url = "https://example.invalid/o/r/pull/1";
20802 runner.state.pr = Some(crate::run::PrRecord {
20803 url: url.into(),
20804 number: 1,
20805 state: "open".into(),
20806 checks: "red".into(),
20807 round: 1,
20808 rounds: 1,
20809 red_at_merge: Vec::new(),
20810 });
20811 runner.state.save().unwrap();
20812 runner.execute_graph().await.unwrap();
20816 assert_eq!(runner.state.status, RunStatus::Blocked);
20817 assert!(runner.state.land_sync_pending);
20818 assert!(runner.state.events.iter().any(|event|
20819 event.node == "land" && event.message == format!("watching {url}")));
20820 assert_eq!(
20821 runner.state.merge.as_ref().unwrap().detail,
20822 "the fixer produced no commit"
20823 );
20824 }
20825 }
20826
20827 #[tokio::test]
20828 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20829 crate::run::pin_test_home();
20830 let tmp = tempfile::tempdir().expect("tempdir");
20831 let repo = tmp.path().join("repo");
20832 std::fs::create_dir_all(&repo).unwrap();
20833 init_repo(&repo);
20834
20835 let mut config = Config::default();
20836 config.merge.mode = MergeMode::Pr;
20837 config.graph.land = true;
20838 config.graph.land_approval = false;
20839
20840 let mut state = RunState::new(
20841 repo.clone(),
20842 "main".to_owned(),
20843 "deadbeef".to_owned(),
20844 "task".to_owned(),
20845 config,
20846 );
20847 state.candidates = vec![Candidate {
20848 index: 0,
20849 label: 'A',
20850 agent: "alpha".to_owned(),
20851 branch: "does-not-exist".to_owned(),
20852 worktree: repo.clone(),
20853 summary: String::new(),
20854 stat: String::new(),
20855 files: 0,
20856 commits: 0,
20857 empty: false,
20858 failed: None,
20859 verified_noop: None,
20860 duration_ms: 0,
20861 folded: false,
20862 }];
20863 state.tally = Some(Tally {
20864 first_choice: BTreeMap::from([('A', 1)]),
20865 borda: BTreeMap::new(),
20866 winner: 'A',
20867 rankings: 1,
20868 unanimous_initial: true,
20869 deliberated: false,
20870 changed_votes: 0,
20871 unanimous_final: true,
20872 tie_break: None,
20873 judges: 0,
20874 present: 0,
20875 quorum: 0,
20876 met_quorum: true,
20877 uncontested: Some("only candidate A produced a change".to_owned()),
20878 });
20879 state.reviews = vec![ReviewRound {
20880 round: 1,
20881 head: "deadbeef".to_owned(),
20882 verified_head: None,
20883 verified_at: None,
20884 reviews: Vec::new(),
20885 e2e: Vec::new(),
20886 fix: None,
20887 blocking: 0,
20888 answered: 0,
20889 expected: 0,
20890 clean: true,
20891 verify_retried: false,
20892 e2e_deferred: false,
20893 e2e_defer_reason: None,
20894 progressed: false,
20895 vote_split: false,
20896 reconsideration: Vec::new(),
20897 verdict: None,
20898 }];
20899 state.gate = vec![CommandOutcome {
20900 command: "test".to_owned(),
20901 code: Some(0),
20902 output_tail: String::new(),
20903 duration_ms: 0,
20904 resource_blocked: false,
20905 }];
20906 state.gate_ran = true;
20907 state.status = RunStatus::Landing;
20911 state.merge = Some(MergeOutcome {
20912 mode: MergeMode::Pr,
20913 ok: true,
20914 detail: "https://example.invalid/x/y/pull/1".to_owned(),
20915 empty: false,
20916 });
20917
20918 ask_test_home();
20922 state.save().expect("save landing run");
20925 let store = ask::Questions::open();
20926 let q = ask_open_question(&store, &state.id);
20927
20928 let mut runner = Runner {
20929 state,
20930 roles: ResolvedRoles {
20931 implementers: Vec::new(),
20932 judges: Vec::new(),
20933 reviewers: Vec::new(),
20934 fixer: None,
20935 conductor: conductor(),
20936 implementer_roster: Vec::new(),
20937 judge_roster: Vec::new(),
20938 reviewer_roster: Vec::new(),
20939 },
20940 sem: Arc::new(Semaphore::new(1)),
20941 pause: Pause::new(),
20942 interrupt: Pause::new(),
20943 };
20944
20945 runner.execute().await.expect("execute");
20950
20951 assert_eq!(
20952 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20953 Some("https://example.invalid/x/y/pull/1"),
20954 "reentry must not push again or open a second pull request over the \
20955 one `land` is already watching"
20956 );
20957 assert_ne!(
20958 runner.state.status,
20959 RunStatus::Landing,
20960 "land could not actually reach the fake pull request, so it must \
20961 have given up rather than left the run silently parked forever"
20962 );
20963 assert_eq!(runner.state.status, RunStatus::Blocked);
20967 assert!(
20968 store.get(&q.id).unwrap().status.open(),
20969 "Blocked is still alive; settle_questions must have been a no-op here"
20970 );
20971 }
20972
20973 fn state_with_round(round: ReviewRound) -> RunState {
20974 let mut s = RunState::new(
20975 PathBuf::from("/repo"),
20976 "main".to_owned(),
20977 "abc1234".to_owned(),
20978 "add retries".to_owned(),
20979 Config::default(),
20980 );
20981 s.reviews = vec![round];
20982 s
20983 }
20984
20985 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
20986 crate::verdict::Finding {
20987 id: id.to_owned(),
20988 severity,
20989 file: None,
20990 line: None,
20991 title: title.to_owned(),
20992 detail: String::new(),
20993 }
20994 }
20995
20996 #[test]
20997 fn pr_body_names_open_findings_and_declined_ones() {
20998 let round = ReviewRound {
20999 round: 2,
21000 head: "deadbee".to_owned(),
21001 verified_head: None,
21002 verified_at: None,
21003 reviews: vec![ReviewRecord {
21004 attempts: 0,
21005 reviewer: 1,
21006 agent: "alpha".to_owned(),
21007 summary: String::new(),
21008 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21009 vote: None,
21010 failed: None,
21011 duration_ms: 0,
21012 }],
21013 e2e: vec![CommandOutcome {
21014 command: "cargo test".to_owned(),
21015 code: Some(0),
21016 output_tail: String::new(),
21017 duration_ms: 0,
21018 resource_blocked: false,
21019 }],
21020 verify_retried: false,
21021 e2e_deferred: false,
21022 e2e_defer_reason: None,
21023 fix: Some(FixRecord {
21024 agent: "alpha".to_owned(),
21025 addressed: Vec::new(),
21026 rejected: vec![crate::verdict::Rejection {
21027 id: "R1-1-1".to_owned(),
21028 why: "not reachable from any caller".to_owned(),
21029 }],
21030 notes: String::new(),
21031 committed: true,
21032 failed: None,
21033 duration_ms: 0,
21034 continuation: None,
21035 }),
21036 blocking: 0,
21037 answered: 1,
21038 expected: 1,
21039 clean: false,
21040 progressed: true,
21041 vote_split: false,
21042 reconsideration: Vec::new(),
21043 verdict: None,
21044 };
21045 let state = state_with_round(round);
21046 let body = pr_message(&state, 'A').body;
21047
21048 assert!(body.contains("add retries"), "the task must still be there");
21049 assert!(body.contains("R2-1-1"), "{body}");
21050 assert!(!body.contains("unused import"), "{body}");
21051 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21052 assert!(
21053 !body.contains("not reachable from any caller"),
21054 "the reason it was declined: {body}"
21055 );
21056 }
21057
21058 #[test]
21059 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21060 let round = ReviewRound {
21061 round: 1,
21062 head: "deadbee".to_owned(),
21063 verified_head: None,
21064 verified_at: None,
21065 reviews: vec![ReviewRecord {
21066 attempts: 0,
21067 reviewer: 1,
21068 agent: "alpha".to_owned(),
21069 summary: String::new(),
21070 findings: Vec::new(),
21071 vote: None,
21072 failed: None,
21073 duration_ms: 0,
21074 }],
21075 e2e: Vec::new(),
21076 verify_retried: false,
21077 e2e_deferred: false,
21078 e2e_defer_reason: None,
21079 fix: None,
21080 blocking: 0,
21081 answered: 1,
21082 expected: 1,
21083 clean: true,
21084 progressed: false,
21085 vote_split: false,
21086 reconsideration: Vec::new(),
21087 verdict: None,
21088 };
21089 let state = state_with_round(round);
21090 let body = pr_message(&state, 'A').body;
21091 assert!(!body.contains("Open review findings"), "{body}");
21092 assert!(!body.contains("Declined"), "{body}");
21093 }
21094
21095 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21096 let mut state = RunState::new(
21097 PathBuf::from("/repo"),
21098 "main".to_owned(),
21099 "abc1234".to_owned(),
21100 instruction.to_owned(),
21101 Config::default(),
21102 );
21103 state.config.disk.min_free_bytes = 0;
21105 state.candidates.push(Candidate {
21106 index: 0,
21107 label: 'A',
21108 agent: "alpha".to_owned(),
21109 branch: "magi/x/A".to_owned(),
21110 worktree: PathBuf::from("/wt"),
21111 summary: summary.to_owned(),
21112 stat: String::new(),
21113 files: 1,
21114 commits: 1,
21115 empty: false,
21116 failed: None,
21117 verified_noop: None,
21118 folded: false,
21119 duration_ms: 0,
21120 });
21121 state
21122 }
21123
21124 fn review_state(subjects: &[&str]) -> RunState {
21125 let mut state = state_with_summary(
21126 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21127 "",
21128 );
21129 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21130 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21131 state
21132 }
21133
21134 #[tokio::test]
21137 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21138 ask_test_home();
21139 let tmp = tempfile::tempdir().unwrap();
21140 let repo = tmp.path().join("repo");
21141 std::fs::create_dir_all(&repo).unwrap();
21142 init_repo(&repo);
21143 let origin = tmp.path().join("origin.git");
21144 let g = |dir: &Path, args: &[&str]| {
21145 let out = std::process::Command::new("git")
21146 .args(args)
21147 .current_dir(dir)
21148 .quiet()
21149 .output()
21150 .expect("spawn git");
21151 assert!(
21152 out.status.success(),
21153 "git {args:?}: {}",
21154 String::from_utf8_lossy(&out.stderr)
21155 );
21156 };
21157 g(
21158 tmp.path(),
21159 &[
21160 "clone",
21161 "--bare",
21162 "-q",
21163 repo.to_str().unwrap(),
21164 origin.to_str().unwrap(),
21165 ],
21166 );
21167 g(
21168 &repo,
21169 &["remote", "add", "origin", origin.to_str().unwrap()],
21170 );
21171 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21172
21173 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21175 g(&repo, &["add", "-A"]);
21176 g(
21177 &repo,
21178 &["commit", "-q", "-m", "chore(deps): update a crate"],
21179 );
21180 g(&repo, &["push", "-q", "origin", "main"]);
21181 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21182 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21183 g(&repo, &["add", "-A"]);
21184 g(
21185 &repo,
21186 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21187 );
21188 g(&repo, &["checkout", "-q", "main"]);
21189
21190 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21191 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21192 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21194 assert_eq!(fallback, c1);
21195
21196 let mut state = review_state(&[
21197 "chore(deps): update a crate",
21198 "fix(daemon): apply a chosen action",
21199 ]);
21200 state.repo = repo.clone();
21201 state.base_branch = "main".to_owned();
21202 state.base_commit = c1;
21203 refresh_reviewed_commits(&mut state, "feat/own").await;
21204 assert_eq!(
21205 state.reviewed_commits,
21206 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21207 );
21208 assert_eq!(
21209 review_title(&state).as_deref(),
21210 Some("fix(daemon): apply a chosen action")
21211 );
21212 assert_eq!(
21213 leaked_subjects(&state, "feat/own").await,
21214 Some(vec!["chore(deps): update a crate".to_owned()])
21215 );
21216 state.config.merge.remote = "nowhere".to_owned();
21219 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21220 }
21221
21222 struct SummaryFixture {
21226 _tmp: tempfile::TempDir,
21227 repo: PathBuf,
21228 reply: PathBuf,
21229 runner: Runner,
21230 }
21231
21232 fn summary_fixture() -> SummaryFixture {
21233 ask_test_home();
21234 let tmp = tempfile::tempdir().unwrap();
21235 let repo = tmp.path().join("repo");
21236 std::fs::create_dir_all(&repo).unwrap();
21237 init_repo(&repo);
21238 let origin = tmp.path().join("origin.git");
21239 let g = |args: &[&str]| {
21240 let out = std::process::Command::new("git")
21241 .args(args)
21242 .current_dir(&repo)
21243 .quiet()
21244 .output()
21245 .expect("spawn git");
21246 assert!(
21247 out.status.success(),
21248 "git {args:?}: {}",
21249 String::from_utf8_lossy(&out.stderr)
21250 );
21251 };
21252 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21253 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21254 g(&["push", "-q", "origin", "main"]);
21255 g(&["checkout", "-q", "-b", "feat/own"]);
21256 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21257 g(&["add", "-A"]);
21258 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21259 let reply = tmp.path().join("reply.txt");
21260 std::fs::write(
21261 &reply,
21262 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21263 )
21264 .unwrap();
21265 let mut runner = runner_at(RunStatus::Gating);
21266 let mut state = review_state(&["feat(own): add the own file"]);
21267 state.repo = repo.clone();
21268 state.base_branch = "main".to_owned();
21269 state.candidates[0].branch = "feat/own".to_owned();
21270 state.candidates[0].worktree = repo.clone();
21271 let mut author = spec("alpha");
21272 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21273 author
21274 .env
21275 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21276 state.config.agents = vec![author];
21277 runner.state = state;
21278 SummaryFixture {
21279 _tmp: tmp,
21280 repo,
21281 reply,
21282 runner,
21283 }
21284 }
21285
21286 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21287 let winner = f.runner.state.candidates[0].clone();
21288 f.runner
21289 .guarded_pr_message(&winner, None, true)
21290 .await
21291 .unwrap()
21292 .expect("an English description never parks")
21293 }
21294
21295 fn clean_round_with_minor_findings() -> ReviewRound {
21296 let mut round = passing_round(2, None);
21297 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21298 round.reviews = vec![ReviewRecord {
21299 attempts: 0,
21300 reviewer: 1,
21301 agent: "alpha".into(),
21302 summary: "内部レビュー".into(),
21303 findings: vec![
21304 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21305 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21306 ],
21307 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21308 failed: None,
21309 duration_ms: 0,
21310 }];
21311 round
21312 }
21313
21314 #[tokio::test]
21315 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21316 for review_only in [false, true] {
21317 let mut f = summary_fixture();
21318 if !review_only {
21319 f.runner.state.candidates[0].agent = "alpha".into();
21320 }
21321 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21322 assert!(f.runner.state.reviews[0].clean);
21323 assert!(f.runner.state.reviews[0].fix.is_none());
21324 assert!(f.runner.state.open_findings().is_empty());
21325 assert_eq!(
21326 crate::pr_outcome::required_finding_ids(&f.runner.state),
21327 ["R2-1-1", "R2-1-2"]
21328 );
21329 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21330 let evidence = crate::pr_outcome::summary_evidence(
21331 &f.runner.state,
21332 crate::pr_outcome::FinalHead::At(&head),
21333 );
21334 assert!(
21335 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21336 );
21337 let omitted = posted(&mut f).await;
21339 assert!(omitted.body.contains("No description"));
21340 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21341 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21342 assert!(f.runner.state.change_summary.is_none());
21343 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21344 let described = posted(&mut f).await;
21345 assert!(described.body.contains("improve documentation for callers"));
21346 assert!(described.body.contains("remaining edge case"));
21347 assert!(described.body.contains("## Open review findings"));
21348 assert!(!described.body.contains("未修正"));
21349 }
21350 }
21351
21352 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21353 let mut f = summary_fixture();
21354 if !review_only {
21355 f.runner.state.candidates[0].agent = "alpha".into();
21356 }
21357 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21358 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21359 fixed
21360 .fix
21361 .as_mut()
21362 .unwrap()
21363 .rejected
21364 .push(crate::verdict::Rejection {
21365 id: "R1-1-1".into(),
21366 why: "呼び出されないコードのため辞退".into(),
21367 });
21368 let mut clean = clean_round_with_minor_findings();
21369 clean.verified_head = Some(head.clone());
21370 f.runner.state.reviews = vec![fixed, clean];
21371 ok_gate(&mut f.runner.state, Some(&head));
21372 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21373 f.runner.state.config.agents.push(scripted("beta", reply));
21374 f.runner.state.config.roles.language_judge =
21375 Some(crate::config::AgentChoice::One("beta".into()));
21376 f
21377 }
21378
21379 #[tokio::test]
21380 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21381 for review_only in [false, true] {
21382 let mut f = rewrite_fixture_with_findings(review_only,
21383 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21384 let m = posted(&mut f).await;
21385 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21386 let expected =
21387 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21388 assert!(m.body.contains(&expected));
21389 assert!(m.body.contains("## Review rounds"));
21390 assert!(
21391 m.body
21392 .contains("Gate (`verify.gate`): passed on the final commit")
21393 );
21394 assert!(
21395 m.body
21396 .contains("Tests (`verify.e2e`): passed on the final commit")
21397 );
21398 let c = f.runner.state.change_summary.as_ref().unwrap();
21399 assert!(
21400 m.body
21401 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21402 );
21403 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21404 assert!(crate::pr_outcome::covers_required_findings(
21405 &f.runner.state,
21406 &m.body
21407 ));
21408 assert!(m.body.contains("documentation remains incomplete"));
21409 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21410 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21411 assert!(f.runner.state.github_text.is_none());
21412 }
21413 }
21414
21415 #[tokio::test]
21416 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21417 for review_only in [false, true] {
21418 for reply in [
21419 "TITLE: fix: public description\nA short English description.",
21420 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21421 ] {
21422 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21423 let winner = f.runner.state.candidates[0].clone();
21424 let posted = f
21425 .runner
21426 .guarded_pr_message(&winner, None, true)
21427 .await
21428 .unwrap();
21429 assert!(posted.is_none());
21430 assert!(f.runner.state.parked);
21431 let withheld = f.runner.state.github_text.as_ref().unwrap();
21432 assert!(
21433 withheld.rewrite.is_none(),
21434 "an incomplete rewrite must not be selectable"
21435 );
21436 let q = ask::Questions::open()
21437 .get(withheld.question.as_ref().unwrap())
21438 .unwrap();
21439 assert!(
21440 !q.choices
21441 .contains(&crate::github_text::USE_REWRITE.to_owned())
21442 );
21443 assert!(
21444 q.detail.contains("R1-1-1")
21445 && q.detail.contains("R2-1-1")
21446 && q.detail.contains("R2-1-2")
21447 );
21448 }
21449 }
21450 }
21451
21452 #[tokio::test]
21453 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21454 for review_only in [false, true] {
21455 let mut f = summary_fixture();
21456 if !review_only {
21457 f.runner.state.candidates[0].agent = "alpha".into();
21458 }
21459 f.runner.state.instruction = "内部タスクの説明".into();
21460 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21461 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21462 f.runner.state.reviews = vec![
21463 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21464 round_with_notes(2, Some("二回目の内部修正報告です。")),
21465 passing_round(3, Some(&head)),
21466 ];
21467 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21468 ok_gate(&mut f.runner.state, Some(&head));
21469 let captured = f.repo.join("public-prompt.txt");
21470 let author = &mut f.runner.state.config.agents[0];
21471 author.command = vec![
21472 "sh".into(),
21473 "-c".into(),
21474 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21475 ];
21476 author.env.insert(
21477 "PROMPT_FILE".into(),
21478 captured.to_string_lossy().into_owned(),
21479 );
21480 let m = posted(&mut f).await;
21481 assert!(m.body.contains("Adds `own.txt`"));
21482 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21483 assert!(m.body.contains("R1-1-1"));
21484 assert!(
21485 m.body
21486 .contains("Gate (`verify.gate`): passed on the final commit")
21487 );
21488 assert!(
21489 m.body
21490 .contains("Tests (`verify.e2e`): passed on the final commit")
21491 );
21492 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21493 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21494 let prompt = std::fs::read_to_string(captured).unwrap();
21495 assert!(prompt.contains(&format!(
21496 "git diff {} {head}",
21497 f.runner.state.change_summary.as_ref().unwrap().base
21498 )));
21499 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21500 assert_eq!(
21501 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21502 "二回目の内部修正報告です。"
21503 );
21504 }
21505 }
21506
21507 #[tokio::test]
21508 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21509 for review_only in [false, true] {
21510 let mut f = summary_fixture();
21511 if !review_only {
21512 f.runner.state.candidates[0].agent = "alpha".into();
21513 }
21514 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21515 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21516 std::fs::write(&f.reply, "no usable description").unwrap();
21517 let failed = posted(&mut f).await;
21518 assert!(
21519 failed
21520 .body
21521 .contains("No description of what this change does could be generated")
21522 );
21523 assert!(failed.body.is_ascii());
21524 assert!(f.runner.state.github_text.is_none());
21525 assert!(f.runner.state.change_summary.is_none());
21526 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21527 let evidence = crate::pr_outcome::summary_evidence(
21528 &f.runner.state,
21529 crate::pr_outcome::FinalHead::At(&head),
21530 );
21531 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21532 assert!(
21533 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21534 .exists()
21535 );
21536 f.runner.state =
21537 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21538 std::fs::write(
21539 &f.reply,
21540 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21541 )
21542 .unwrap();
21543 let retried = posted(&mut f).await;
21544 assert!(retried.body.contains("Adds the committed own file."));
21545 assert_eq!(
21546 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21547 "日本語の修正報告"
21548 );
21549 }
21550 }
21551
21552 #[tokio::test]
21553 async fn same_head_verification_and_findings_invalidate_public_prose() {
21554 let mut f = summary_fixture();
21555 posted(&mut f).await;
21556 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21557 let original = f
21558 .runner
21559 .state
21560 .change_summary
21561 .as_ref()
21562 .unwrap()
21563 .evidence
21564 .clone();
21565 ok_gate(&mut f.runner.state, Some(&head));
21566 std::fs::write(
21567 &f.reply,
21568 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21569 )
21570 .unwrap();
21571 let verified = posted(&mut f).await;
21572 assert!(verified.body.contains("now verified"));
21573 assert_ne!(
21574 f.runner.state.change_summary.as_ref().unwrap().evidence,
21575 original
21576 );
21577 let mut round = round_with_notes(1, Some("内部報告"));
21578 round.reviews.push(ReviewRecord {
21579 attempts: 0,
21580 reviewer: 1,
21581 agent: "alpha".into(),
21582 summary: "内部レビュー".into(),
21583 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21584 vote: None,
21585 failed: None,
21586 duration_ms: 0,
21587 });
21588 round
21589 .fix
21590 .as_mut()
21591 .unwrap()
21592 .rejected
21593 .push(crate::verdict::Rejection {
21594 id: "R1-1-1".into(),
21595 why: "呼び出し元から到達できない".into(),
21596 });
21597 f.runner.state.reviews = vec![round];
21598 let omitted = posted(&mut f).await;
21600 assert!(omitted.body.contains("No description"));
21601 assert!(omitted.body.contains("R1-1-1"));
21602 assert!(!omitted.body.contains("到達"));
21603 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21604 let described = posted(&mut f).await;
21605 assert!(described.body.contains("unreachable from callers"));
21606 assert!(described.body.contains("R1-1-1"));
21607 assert!(!described.body.contains("到達"));
21608 let evidence = f
21609 .runner
21610 .state
21611 .change_summary
21612 .as_ref()
21613 .unwrap()
21614 .evidence
21615 .clone();
21616 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21617 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21618 assert!(posted(&mut f).await.body.contains("rationale updated"));
21619 assert_ne!(
21620 f.runner.state.change_summary.as_ref().unwrap().evidence,
21621 evidence
21622 );
21623 f.runner
21625 .state
21626 .change_summary
21627 .as_mut()
21628 .unwrap()
21629 .evidence
21630 .clear();
21631 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21632 assert!(posted(&mut f).await.body.contains("current specification"));
21633 }
21634
21635 #[tokio::test]
21636 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21637 let mut f = summary_fixture();
21638 let m = posted(&mut f).await;
21639 assert_eq!(m.title, "feat(own): add the own file");
21640 assert!(
21641 m.body
21642 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21643 "{}",
21644 m.body
21645 );
21646 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21647 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21648 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21649 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21650 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21651 let kept = f.runner.state.change_summary.as_ref().unwrap();
21652 assert_eq!(
21653 (kept.head.as_str(), kept.base.as_str()),
21654 (head.as_str(), base.as_str())
21655 );
21656 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21657 }
21658
21659 #[tokio::test]
21660 async fn review_only_summary_receives_the_configured_github_language() {
21661 let mut f = summary_fixture();
21662 f.runner.state.config.graph.github_language = "ja".into();
21663 let captured = f.repo.join("summary-prompt.txt");
21664 let author = &mut f.runner.state.config.agents[0];
21665 author.command = vec![
21666 "sh".into(),
21667 "-c".into(),
21668 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21669 ];
21670 author.env.insert(
21671 "PROMPT_FILE".into(),
21672 captured.to_string_lossy().into_owned(),
21673 );
21674 std::fs::write(
21675 &f.reply,
21676 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21677 )
21678 .unwrap();
21679 let m = posted(&mut f).await;
21680 let prompt = std::fs::read_to_string(captured).unwrap();
21681 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21682 assert!(!prompt.contains("in English"));
21683 assert_eq!(m.title, "feat(own): ファイルを追加");
21684 assert!(m.body.contains("own.txt を追加します。"));
21685 }
21686
21687 #[tokio::test]
21688 async fn review_fixes_come_after_the_whole_change_summary() {
21689 let mut f = summary_fixture();
21690 f.runner.state.reviews = vec![
21691 round_with_notes(1, Some("handled the PATH case")),
21692 round_with_notes(2, None),
21693 ];
21694 let m = posted(&mut f).await;
21695 let summary = m.body.find("## Summary").expect("summary");
21696 let fixes = m.body.find("## Review rounds").expect("review rounds");
21697 assert!(summary < fixes, "{}", m.body);
21698 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21699 assert!(m.body.contains("## Final verification"), "{}", m.body);
21700 }
21701
21702 #[tokio::test]
21703 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21704 let mut f = summary_fixture();
21705 let first = posted(&mut f).await;
21706 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21707 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21709 let again = posted(&mut f).await;
21710 assert_eq!((again.title, again.body), (first.title, first.body));
21711
21712 let g = |args: &[&str]| {
21713 let out = std::process::Command::new("git")
21714 .args(args)
21715 .current_dir(&f.repo)
21716 .quiet()
21717 .output()
21718 .expect("spawn git");
21719 assert!(out.status.success(), "git {args:?}");
21720 };
21721 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21722 g(&["add", "-A"]);
21723 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21724 let second = posted(&mut f).await;
21725 let kept = f.runner.state.change_summary.clone().unwrap();
21726 assert_ne!(kept.head, first_head);
21727 assert!(second.body.contains("Not read yet."), "{}", second.body);
21728 assert!(!second.body.contains("so the first summary is read"));
21729 assert_eq!(second.title, "feat(own): other");
21730 }
21731
21732 #[tokio::test]
21733 async fn an_unavailable_summary_is_stated_and_never_invented() {
21734 let mut f = summary_fixture();
21735 std::fs::write(&f.reply, "no title line here").unwrap();
21736 let m = posted(&mut f).await;
21737 assert!(
21738 m.body
21739 .contains("No description of what this change does could be generated"),
21740 "{}",
21741 m.body
21742 );
21743 assert!(m.body.contains("internal reports remain in local records"));
21744 assert!(f.runner.state.change_summary.is_none());
21745 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21746 let winner = f.runner.state.candidates[0].clone();
21748 let preview = f
21749 .runner
21750 .guarded_pr_message(&winner, None, false)
21751 .await
21752 .unwrap()
21753 .unwrap();
21754 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21755 }
21756
21757 #[test]
21758 fn pr_message_review_single_commit_uses_its_subject() {
21759 let state = review_state(&["feat(nats): per-role user"]);
21760 let m = pr_message(&state, 'A');
21761 assert_eq!(m.title, "feat(nats): per-role user");
21762 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21763 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21764 }
21765
21766 #[test]
21767 fn pr_message_review_multi_commit_takes_the_oldest() {
21768 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21769 let m = pr_message(&state, 'A');
21770 assert_eq!(m.title, "feat: the change");
21771 for s in ["feat: the change", "fix: typo", "fix: again"] {
21772 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21773 }
21774 }
21775
21776 #[test]
21777 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21778 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21779 let mut state = review_state(&[first, "fix: later fixup"]);
21780 state.candidates[0].branch = "機能/ログイン".to_owned();
21781 let m = pr_message(&state, 'A');
21782 assert!(
21783 m.title.starts_with("chore: land candidate A of run"),
21784 "{}",
21785 m.title
21786 );
21787 }
21788 }
21789
21790 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21791 BranchFacts {
21792 commits: commits
21793 .iter()
21794 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21795 .collect(),
21796 stat: stat.to_owned(),
21797 }
21798 }
21799
21800 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21801 let mut r = review_round(true, 0, 1, 1, true, true);
21802 r.round = round;
21803 r.fix = notes.map(|n| FixRecord {
21804 agent: "fixer".to_owned(),
21805 addressed: Vec::new(),
21806 rejected: Vec::new(),
21807 notes: n.to_owned(),
21808 committed: true,
21809 failed: None,
21810 duration_ms: 0,
21811 continuation: None,
21812 });
21813 r
21814 }
21815
21816 #[test]
21817 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21818 let state = review_state(&["ignored"]);
21819 let f = facts(
21820 &[
21821 (
21822 "fix(login): resolve PATH on macOS",
21823 "Login shells skip rc files.",
21824 ),
21825 ("fix: address review", ""),
21826 ],
21827 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21828 );
21829 let m = pr_message_with(&state, 'A', Some(&f));
21830 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21831 assert!(
21832 m.body
21833 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
21834 "{}",
21835 m.body
21836 );
21837 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21838 assert!(m.body.contains("## Diff stat"), "{}", m.body);
21839 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21840 for banned in [
21841 "Review the work already",
21842 "no task statement",
21843 "Original task",
21844 ] {
21845 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21846 }
21847 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
21848 }
21849
21850 #[test]
21851 fn pr_message_review_truncates_a_huge_first_commit_body() {
21852 let state = review_state(&["ignored"]);
21853 let f = facts(
21854 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
21855 "s",
21856 );
21857 let m = pr_message_with(&state, 'A', Some(&f));
21858 assert!(m.body.len() < 30_000, "{}", m.body.len());
21859 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
21860 assert!(
21861 m.body.contains("- ... 1 more commit(s)"),
21862 "{}",
21863 m.body.len()
21864 );
21865 assert!(m.body.ends_with("magi:candidate-a\n"));
21866 }
21867
21868 #[test]
21869 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
21870 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
21871 assert_eq!(m.title, "feat: x");
21872 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
21873 assert!(!m.body.contains("Diff stat"), "{}", m.body);
21874 assert!(!m.body.contains("no task statement"), "{}", m.body);
21875 }
21876
21877 #[test]
21878 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
21879 let mut state = review_state(&["日本語の件名"]);
21880 state.candidates[0].branch = "fix/macos-login-path".to_owned();
21881 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
21882 state.candidates[0].branch = "機能/ログイン".to_owned();
21883 assert!(
21884 pr_message(&state, 'A')
21885 .title
21886 .starts_with("chore: land candidate A")
21887 );
21888 }
21889
21890 #[test]
21891 fn pr_message_review_fixes_survive_a_clean_final_round() {
21892 let mut state = review_state(&["feat: x"]);
21893 state.reviews = vec![
21894 round_with_notes(1, Some("handled the PATH case")),
21895 round_with_notes(2, None),
21896 ];
21897 let body = pr_message(&state, 'A').body;
21898 assert!(
21899 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
21900 "{body}"
21901 );
21902 assert!(body.contains("### Round 1"), "{body}");
21903
21904 state.reviews = vec![
21905 round_with_notes(1, Some("first fix")),
21906 round_with_notes(2, Some("")),
21907 round_with_notes(3, Some("second fix")),
21908 round_with_notes(4, None),
21909 ];
21910 let body = pr_message(&state, 'A').body;
21911 assert!(
21912 body.contains("### Round 1") && !body.contains("first fix"),
21913 "{body}"
21914 );
21915 assert!(
21916 body.contains("### Round 3") && !body.contains("second fix"),
21917 "{body}"
21918 );
21919 assert!(body.contains("### Round 2"), "{body}");
21920 }
21921
21922 const FINAL: &str = "abcdef1234567890";
21923
21924 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
21925 let mut r = round_with_notes(round, None);
21926 r.verified_head = verified.map(str::to_owned);
21927 r
21928 }
21929
21930 fn ok_gate(state: &mut RunState, head: Option<&str>) {
21931 state.gate = vec![CommandOutcome {
21932 command: "gate".to_owned(),
21933 code: Some(0),
21934 output_tail: String::new(),
21935 duration_ms: 0,
21936 resource_blocked: false,
21937 }];
21938 state.gate_ran = true;
21939 state.gate_head = head.map(str::to_owned);
21940 }
21941
21942 fn body_at(state: &RunState) -> String {
21943 pr_message_raw(
21944 state,
21945 'A',
21946 None,
21947 crate::pr_outcome::FinalHead::At(FINAL),
21948 ChangeNote::NotAsked,
21949 )
21950 .body
21951 }
21952
21953 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
21954
21955 #[test]
21956 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
21957 let mut state = review_state(&["feat: x"]);
21958 state.reviews = vec![
21959 round_with_notes(1, Some(SANDBOXED)),
21960 passing_round(2, Some(FINAL)),
21961 ];
21962 ok_gate(&mut state, Some(FINAL));
21963 let body = body_at(&state);
21964 assert!(!body.contains("Handled the PATH case."), "{body}");
21965 assert!(!body.contains("could not commit"), "{body}");
21966 assert!(!body.contains("could not run"), "{body}");
21967 assert!(
21968 body.contains("The fix is committed on the branch"),
21969 "{body}"
21970 );
21971 assert!(!body.contains("sandbox"), "{body}");
21972 assert!(
21973 body.contains("Tests (`verify.e2e`): passed on the final commit"),
21974 "{body}"
21975 );
21976 assert!(
21977 body.contains("Gate (`verify.gate`): passed on the final commit"),
21978 "{body}"
21979 );
21980 assert!(
21982 state.reviews[0]
21983 .fix
21984 .as_ref()
21985 .unwrap()
21986 .notes
21987 .contains("could not commit")
21988 );
21989 }
21990
21991 #[test]
21992 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
21993 let mut state = review_state(&["feat: x"]);
21994 let mut r1 = round_with_notes(1, Some(SANDBOXED));
21995 r1.fix.as_mut().unwrap().committed = false;
21996 let mut r2 = passing_round(2, Some(FINAL));
21997 r2.e2e[0].code = Some(1);
21998 state.reviews = vec![r1, r2];
21999 let body = body_at(&state);
22000 assert!(!body.contains("could not commit"), "{body}");
22001 assert!(!body.contains("could not run"), "{body}");
22002 assert!(body.contains("This round produced no commit"), "{body}");
22003 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22004 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22005 assert!(!body.contains("passed"), "{body}");
22006 }
22007
22008 #[test]
22009 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22010 let mut state = review_state(&["feat: x"]);
22011 state.reviews = vec![
22012 round_with_notes(1, Some(SANDBOXED)),
22013 passing_round(2, Some("0123456789abcdef")),
22014 ];
22015 ok_gate(&mut state, Some("0123456789abcdef"));
22016 let body = body_at(&state);
22017 assert!(!body.contains("HTTP tests could not run"), "{body}");
22018 assert!(!body.contains("could not commit"), "{body}");
22020 assert!(
22021 body.contains("passed on an earlier commit (`0123456`)"),
22022 "{body}"
22023 );
22024 assert!(!body.contains("passed on the final commit"), "{body}");
22025
22026 ok_gate(&mut state, None);
22028 let body = body_at(&state);
22029 assert!(
22030 body.contains("passed on a commit that was not recorded"),
22031 "{body}"
22032 );
22033 }
22034
22035 #[test]
22036 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22037 let state = state_with_summary(
22038 "add retries to the client",
22039 "TITLE: feat: retries\n\nDid it.",
22040 );
22041 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22042 assert_eq!(m.title, "feat: retries");
22043 assert!(
22044 m.body.contains("<summary>Original task</summary>"),
22045 "{}",
22046 m.body
22047 );
22048 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22049 assert!(
22050 m.body
22051 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22052 "{}",
22053 m.body
22054 );
22055 }
22056
22057 #[test]
22058 fn pr_message_review_bounds_a_long_english_subject() {
22059 let long = format!("feat: {}", "word ".repeat(100));
22060 let m = pr_message(&review_state(&[&long]), 'A');
22061 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22062 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22063 }
22064
22065 #[test]
22066 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22067 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22068 let m = pr_message(&state, 'A');
22069 assert_eq!(m.title, "add retries");
22070 assert!(m.body.contains("<summary>Original task</summary>"));
22071 assert!(!m.body.contains("Commits under review"));
22072 assert_eq!(landing_subject_source(&state), state.instruction);
22073 }
22074
22075 #[test]
22076 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22077 let state = review_state(&["feat: the change", "fix: typo"]);
22078 let source = landing_subject_source(&state);
22079 assert_eq!(land::merge_subject("", &source), "feat: the change");
22080 assert_eq!(
22081 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22082 "feat: the change"
22083 );
22084 assert_eq!(
22086 land::merge_subject("feat: renamed by hand", &source),
22087 "feat: renamed by hand"
22088 );
22089 let blank = review_state(&["日本語"]);
22090 assert!(
22091 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22092 );
22093 }
22094
22095 #[test]
22096 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22097 let state = review_state(&["feat: the change"]);
22098 let source = landing_subject_source(&state);
22099 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22100 assert_eq!(
22101 land::merge_subject(landing_title(&state, old), &source),
22102 "feat: the change"
22103 );
22104 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22105 let task = state_with_summary("add retries", "");
22106 assert_eq!(landing_title(&task, old), old);
22107 }
22108
22109 #[test]
22110 fn pr_message_describes_the_change_not_the_task() {
22111 let state = state_with_summary(
22112 "今回やってほしいこと: results projector を直す",
22113 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22114 );
22115 let m = pr_message(&state, 'A');
22116 assert_eq!(m.title, "fix(web): batch the runs list reads");
22117 assert!(
22118 m.body.starts_with("## Summary\n\n- reads run.json once"),
22119 "{}",
22120 m.body
22121 );
22122 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22123 let task_at = m.body.find("今回やってほしいこと").unwrap();
22124 let details_at = m.body.find("<details>").unwrap();
22125 assert!(
22126 details_at < task_at,
22127 "the task lives inside <details>: {}",
22128 m.body
22129 );
22130 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22131 assert!(m.body.contains("magi:candidate-a"));
22132 }
22133
22134 #[test]
22135 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22136 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22137 let m = pr_message(&state, 'A');
22138 assert_eq!(m.title, "add retries");
22139 assert!(
22140 m.body.contains("## Summary\n\n- did some things"),
22141 "{}",
22142 m.body
22143 );
22144
22145 let none = RunState::new(
22146 PathBuf::from("/repo"),
22147 "main".to_owned(),
22148 "abc1234".to_owned(),
22149 "add retries".to_owned(),
22150 Config::default(),
22151 );
22152 let m = pr_message(&none, 'A');
22153 assert_eq!(m.title, "add retries");
22154 assert!(!m.body.contains("## Summary"), "{}", m.body);
22155 }
22156
22157 #[test]
22158 fn pr_message_refuses_the_candidate_commit_subject() {
22159 for bad in [
22160 "TITLE: magi: candidate A (uncommitted work)",
22161 "TITLE: chore: stuff (uncommitted work)",
22162 "TITLE: ",
22163 ] {
22164 let state = state_with_summary("add retries", bad);
22165 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22166 }
22167 }
22168
22169 #[test]
22170 fn pr_message_bounds_a_very_long_task_and_title() {
22171 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22172 let state = state_with_summary(&long, "- nothing");
22173 let m = pr_message(&state, 'A');
22174 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22175 assert!(!m.title.contains('\n'));
22176
22177 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22178 let m = pr_message(&state, 'A');
22179 assert!(m.title.starts_with("feat: "));
22180 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22181 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22182 }
22183
22184 fn long_title_of(instruction: &str) -> String {
22185 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22186 }
22187
22188 #[test]
22189 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22190 let first = "Make the landing path keep a readable title for long tasks";
22191 let line = format!(
22192 "{first}. {}",
22193 "And then keep going with more words ".repeat(20)
22194 );
22195 let t = long_title_of(&line);
22196 assert_eq!(t, first);
22197 assert!(!t.starts_with("chore: land"));
22198 }
22199
22200 #[test]
22201 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22202 let line = "word ".repeat(200);
22203 let t = long_title_of(&line);
22204 assert!(t.ends_with("word..."), "{t}");
22205 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22206 }
22207
22208 #[test]
22209 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22210 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22211 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22212 }
22213 }
22214
22215 #[test]
22216 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22217 let mut state = state_with_summary("再試行を追加する", "- nothing");
22218 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22219 state.config.graph.github_language = "ja".into();
22220 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22221 }
22222
22223 #[test]
22224 fn pr_message_title_limit_is_exact() {
22225 let at = "a".repeat(PR_TITLE_MAX);
22226 assert_eq!(long_title_of(&at), at);
22227 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22228 assert!(over.ends_with("..."), "{over}");
22229 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22230 }
22231
22232 #[test]
22233 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22234 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22235 let t = long_title_of(&line);
22236 assert!(t.ends_with("..."), "{t}");
22237 assert!(t.is_ascii(), "{t}");
22238 }
22239
22240 #[test]
22241 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22242 let line = format!(
22243 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22244 "plus more filler words ".repeat(20)
22245 );
22246 let t = long_title_of(&line);
22247 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22248 }
22249
22250 #[test]
22251 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22252 let line = format!("\n\n# {}", "title words ".repeat(40));
22253 let state = state_with_summary(&line, "- nothing");
22254 let m = pr_message(&state, 'A');
22255 assert!(m.title.starts_with("title words"), "{}", m.title);
22256 assert_eq!(
22257 land::merge_subject(&m.title, &landing_subject_source(&state)),
22258 m.title
22259 );
22260 assert_eq!(
22262 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22263 "feat: renamed by hand"
22264 );
22265 }
22266
22267 #[tokio::test]
22268 async fn github_text_rewrite_is_bounded_and_falls_back() {
22269 crate::run::pin_test_home();
22270 for (reply, accepted) in [
22271 (
22272 "TITLE: fix: retries\nAdd retries for failed requests.",
22273 true,
22274 ),
22275 (
22276 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22277 false,
22278 ),
22279 (
22280 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22281 true,
22282 ),
22283 (
22284 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22285 true,
22286 ),
22287 (
22288 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22289 true,
22290 ),
22291 (
22292 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22293 false,
22294 ),
22295 ("TITLE: fix: retries\n", false),
22296 ] {
22297 let dir = tempfile::tempdir().unwrap();
22298 let mut runner = runner_at(RunStatus::Gating);
22299 runner.state = state_with_summary(
22300 "add retries",
22301 "TITLE: fix: retries\n日本語の説明を書きます。",
22302 );
22303 runner.state.repo = dir.path().to_owned();
22304 runner.state.candidates[0].worktree = dir.path().to_owned();
22305 let mut author = spec("alpha");
22306 author.command = vec![
22307 "sh".into(),
22308 "-c".into(),
22309 "printf '%s' \"$REWRITE_REPLY\"".into(),
22310 ];
22311 author.env.insert("REWRITE_REPLY".into(), reply.into());
22312 runner.state.config.agents = vec![author];
22313 seed_public_summary(&mut runner);
22314 let winner = runner.state.candidates[0].clone();
22315 let first = runner
22316 .guarded_pr_message(&winner, None, true)
22317 .await
22318 .unwrap();
22319 let message = if accepted {
22320 first.expect("a passing rewrite never asks")
22321 } else {
22322 assert!(first.is_none());
22324 assert!(runner.state.parked);
22325 let store = ask::Questions::open();
22326 let asked: Vec<_> = store
22327 .list()
22328 .into_iter()
22329 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22330 .collect();
22331 assert_eq!(asked.len(), 1);
22332 assert!(!asked[0].detail.contains("secret"));
22333 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22334 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22335 assert_eq!(
22336 asked[0]
22337 .choices
22338 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22339 rewrite.is_some()
22340 );
22341 store
22344 .update(&asked[0].id, |q| {
22345 q.abandon("test");
22346 Ok(())
22347 })
22348 .unwrap();
22349 let again = runner
22350 .guarded_pr_message(&winner, None, true)
22351 .await
22352 .unwrap()
22353 .expect("an abandoned question falls back");
22354 let n = store
22355 .list()
22356 .into_iter()
22357 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22358 .count();
22359 assert_eq!(n, 1, "asked once per rejected text");
22360 again
22361 };
22362 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22363 assert!(!message.body.contains("secret"));
22364 assert!(!message.body.contains("prefix"));
22365 assert!(!message.title.contains("secret"));
22366 assert_eq!(
22367 message.body.contains("Add retries for failed requests."),
22368 accepted
22369 );
22370 assert_eq!(
22371 runner.state.seats["github-text-rewrite"].turns, 1,
22372 "only one rewrite invocation"
22373 );
22374 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22375 if accepted {
22376 assert!(runner.state.github_text.is_none());
22377 assert!(!runner.state.parked);
22378 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22379 assert!(message.body.contains("## Final verification"));
22380 assert!(message.body.contains("Evidence: final diff"));
22381 assert!(
22382 message
22383 .body
22384 .contains(&format!("magi:run/{}", runner.state.id))
22385 );
22386 }
22387 if !accepted {
22388 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22389 assert!(
22390 message
22391 .body
22392 .contains(&format!("magi:run/{}", runner.state.id))
22393 );
22394 }
22395 }
22396 }
22397
22398 fn scripted(id: &str, reply: &str) -> AgentSpec {
22399 let mut a = spec(id);
22400 a.command = vec![
22401 "sh".into(),
22402 "-c".into(),
22403 "printf '%s' \"$REWRITE_REPLY\"".into(),
22404 ];
22405 a.env.insert("REWRITE_REPLY".into(), reply.into());
22406 a
22407 }
22408
22409 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22410 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22411
22412 fn seed_public_summary(runner: &mut Runner) {
22414 let repo = &runner.state.repo;
22415 let g = |args: &[&str]| {
22416 let out = std::process::Command::new("git")
22417 .args(args)
22418 .current_dir(repo)
22419 .quiet()
22420 .output()
22421 .unwrap();
22422 assert!(out.status.success(), "git {args:?}");
22423 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22424 };
22425 g(&["init", "-q", "-b", "main"]);
22426 g(&[
22427 "-c",
22428 "user.name=Test",
22429 "-c",
22430 "user.email=test@example.invalid",
22431 "commit",
22432 "-q",
22433 "--allow-empty",
22434 "-m",
22435 "feat: fixture",
22436 ]);
22437 let head = g(&["rev-parse", "HEAD"]);
22438 g(&["update-ref", "refs/remotes/origin/main", &head]);
22439 let evidence = crate::pr_outcome::summary_evidence(
22440 &runner.state,
22441 crate::pr_outcome::FinalHead::At(&head),
22442 );
22443 let generated = &runner.state.candidates[0].summary;
22444 runner.state.change_summary = Some(crate::run::ChangeSummary {
22445 base: head.clone(),
22446 head,
22447 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22448 title: summary_title(generated).unwrap(),
22449 body: summary_without_title(generated),
22450 });
22451 }
22452
22453 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22454 let mut runner = runner_at(RunStatus::Gating);
22455 runner.state = state_with_summary("add retries", JA_SUMMARY);
22456 if review_only {
22457 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22458 }
22459 runner.state.repo = dir.to_owned();
22460 runner.state.candidates[0].worktree = dir.to_owned();
22461 runner.state.config.agents = agents;
22462 seed_public_summary(&mut runner);
22463 runner
22464 }
22465
22466 fn gate_questions(runner: &Runner) -> usize {
22467 ask::Questions::open()
22468 .list()
22469 .into_iter()
22470 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22471 .count()
22472 }
22473
22474 #[tokio::test]
22475 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22476 crate::run::pin_test_home();
22477 let dir = tempfile::tempdir().unwrap();
22478 let mut runner = rewrite_runner(
22479 dir.path(),
22480 true,
22481 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22482 );
22483 runner.state.config.roles.language_judge =
22484 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22485 let winner = runner.state.candidates[0].clone();
22486 let msg = runner
22487 .guarded_pr_message(&winner, None, true)
22488 .await
22489 .unwrap()
22490 .expect("an English rewrite posts without asking");
22491 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22492 assert!(msg.body.contains("<details>kept</details>"));
22493 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22494 assert_eq!(gate_questions(&runner), 0);
22495 assert!(runner.state.github_text.is_none());
22496 assert!(!runner.state.seats.contains_key("impl-A"));
22497 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22498 }
22499
22500 #[tokio::test]
22501 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22502 crate::run::pin_test_home();
22503 let dir = tempfile::tempdir().unwrap();
22504 let mut runner = rewrite_runner(
22505 dir.path(),
22506 false,
22507 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22508 );
22509 runner.state.config.roles.language_judge =
22510 Some(crate::config::AgentChoice::One("beta".into()));
22511 runner.state.seats.insert(
22512 "impl-A".into(),
22513 SeatState::new("impl-A", "alpha", runner.state.seed),
22514 );
22515 let winner = runner.state.candidates[0].clone();
22516 let msg = runner
22517 .guarded_pr_message(&winner, None, true)
22518 .await
22519 .unwrap()
22520 .expect("rewritten");
22521 assert!(msg.body.contains("<details>kept</details>"));
22522 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22523 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22524 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22525 }
22526
22527 #[tokio::test]
22528 async fn rewrite_reuses_a_resumable_implementer_session() {
22529 crate::run::pin_test_home();
22530 let dir = tempfile::tempdir().unwrap();
22531 let mut runner = rewrite_runner(
22532 dir.path(),
22533 false,
22534 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22535 );
22536 runner.state.config.roles.language_judge =
22537 Some(crate::config::AgentChoice::One("beta".into()));
22538 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22539 seat.turns = 1;
22540 runner.state.seats.insert("impl-A".into(), seat);
22541 let winner = runner.state.candidates[0].clone();
22542 runner
22543 .guarded_pr_message(&winner, None, true)
22544 .await
22545 .unwrap()
22546 .expect("rewritten");
22547 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22548 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22549 }
22550
22551 #[tokio::test]
22552 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22553 crate::run::pin_test_home();
22556 let dir = tempfile::tempdir().unwrap();
22557 let mut exhausted = spec("alpha");
22558 exhausted.command = vec!["false".into()];
22559 let mut runner = rewrite_runner(
22560 dir.path(),
22561 false,
22562 vec![exhausted, scripted("beta", EN_REWRITE)],
22563 );
22564 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22566 "alpha".into(),
22567 "beta".into(),
22568 ]));
22569 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22570 seat.turns = 1;
22571 runner.state.seats.insert("impl-A".into(), seat);
22572 let winner = runner.state.candidates[0].clone();
22573 let msg = runner
22574 .guarded_pr_message(&winner, None, true)
22575 .await
22576 .unwrap()
22577 .expect("the role rewrites after the implementer fails");
22578 assert!(msg.body.contains("`src/net.rs`"));
22579 assert_eq!(
22580 runner.state.seats["impl-A"].turns, 2,
22581 "the implementer was tried once"
22582 );
22583 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22584 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22585 }
22586
22587 #[tokio::test]
22588 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22589 crate::run::pin_test_home();
22590 let dir = tempfile::tempdir().unwrap();
22591 let mut broken = spec("bad");
22592 broken.command = vec!["false".into()];
22593 let mut runner = rewrite_runner(
22594 dir.path(),
22595 true,
22596 vec![
22597 broken,
22598 scripted("noformat", "Add retries."),
22599 scripted("good", EN_REWRITE),
22600 ],
22601 );
22602 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22603 "bad".into(),
22604 "noformat".into(),
22605 "good".into(),
22606 ]));
22607 let winner = runner.state.candidates[0].clone();
22608 let msg = runner
22609 .guarded_pr_message(&winner, None, true)
22610 .await
22611 .unwrap()
22612 .expect("the third agent rewrites");
22613 assert!(msg.body.contains("`src/net.rs`"));
22614 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22615 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22616 }
22617
22618 #[tokio::test]
22619 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22620 crate::run::pin_test_home();
22621 for (agents, role, saved) in [
22623 (
22624 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22625 Some("beta"),
22626 true,
22627 ),
22628 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22629 (Vec::new(), None, false),
22630 ] {
22631 let dir = tempfile::tempdir().unwrap();
22632 let mut runner = rewrite_runner(dir.path(), true, agents);
22633 runner.state.config.roles.language_judge =
22634 role.map(|r| crate::config::AgentChoice::One(r.into()));
22635 let winner = runner.state.candidates[0].clone();
22636 let first = runner
22637 .guarded_pr_message(&winner, None, true)
22638 .await
22639 .unwrap();
22640 assert!(first.is_none(), "withheld text parks the run");
22641 assert!(runner.state.parked);
22642 assert_eq!(gate_questions(&runner), 1);
22643 let g = runner.state.github_text.as_ref().unwrap();
22644 assert_eq!(g.rewrite.is_some(), saved);
22645 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22646 }
22647 }
22648
22649 #[tokio::test]
22650 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22651 crate::run::pin_test_home();
22652 let dir = tempfile::tempdir().unwrap();
22653 let mut runner = runner_at(RunStatus::Gating);
22654 runner.state =
22655 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22656 runner.state.repo = dir.path().to_owned();
22657 runner.state.candidates[0].worktree = dir.path().to_owned();
22658 let mut author = spec("alpha");
22659 author.command = vec![
22660 "sh".into(),
22661 "-c".into(),
22662 "printf '%s' \"$REWRITE_REPLY\"".into(),
22663 ];
22664 author.env.insert(
22665 "REWRITE_REPLY".into(),
22666 format!(
22667 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22668 "English sentence. ".repeat(600)
22669 ),
22670 );
22671 runner.state.config.agents = vec![author];
22672 seed_public_summary(&mut runner);
22673 let winner = runner.state.candidates[0].clone();
22674 assert!(
22675 runner
22676 .guarded_pr_message(&winner, None, true)
22677 .await
22678 .unwrap()
22679 .is_none()
22680 );
22681 let record = runner.state.github_text.clone().unwrap();
22682 let rewrite = record.rewrite.clone().unwrap();
22683 assert_eq!(rewrite.failed, ["body-language"]);
22684 assert!(
22685 rewrite
22686 .body
22687 .ends_with(&format!("magi:run/{}", runner.state.id))
22688 );
22689 let store = ask::Questions::open();
22690 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22691 assert!(
22692 q.choices
22693 .contains(&crate::github_text::USE_REWRITE.to_owned())
22694 );
22695 for label in [
22696 "ORIGINAL",
22697 "FALLBACK",
22698 "REWRITE",
22699 "Rules still failed: body-language",
22700 "truncated",
22701 ] {
22702 assert!(q.detail.contains(label), "{label}");
22703 }
22704 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22705 &runner.state,
22706 &crate::github_text::artifact_name(&record.fingerprint),
22707 ))
22708 .unwrap();
22709 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22710 runner.state =
22712 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22713 store
22714 .update(&q.id, |q| {
22715 q.say("まだ日本語")?;
22716 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22717 })
22718 .unwrap();
22719 assert!(
22720 runner
22721 .guarded_pr_message(&winner, None, true)
22722 .await
22723 .unwrap()
22724 .is_none()
22725 );
22726 let retry = runner.state.github_text.as_ref().unwrap();
22727 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22728 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22729 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22730 assert!(
22731 q.posting_comparison
22732 .as_ref()
22733 .unwrap()
22734 .matches_detail(&q.detail)
22735 );
22736 store
22737 .update(&q.id, |q| {
22738 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22739 })
22740 .unwrap();
22741 let message = runner
22742 .guarded_pr_message(&winner, None, true)
22743 .await
22744 .unwrap()
22745 .unwrap();
22746 assert_eq!(message.title, rewrite.title);
22747 assert_eq!(message.body, rewrite.body);
22748 assert_eq!(
22749 runner.state.github_text.as_ref().unwrap().choice,
22750 Some(GithubTextChoice::Rewrite)
22751 );
22752 let again = runner
22753 .guarded_pr_message(&winner, None, true)
22754 .await
22755 .unwrap()
22756 .unwrap();
22757 assert_eq!(again.title, message.title);
22758 assert_eq!(again.body, message.body);
22759 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22760 assert!(
22761 !runner
22762 .state
22763 .events
22764 .iter()
22765 .any(|e| e.message.contains(&rewrite.title)
22766 || e.message.contains("English sentence."))
22767 );
22768 }
22769
22770 #[tokio::test]
22771 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22772 crate::run::pin_test_home();
22773 let dir = tempfile::tempdir().unwrap();
22774 let mut runner = runner_at(RunStatus::Gating);
22775 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22776 runner.state.repo = dir.path().to_owned();
22777 runner.state.candidates[0].worktree = dir.path().to_owned();
22778 seed_public_summary(&mut runner);
22779 let winner = runner.state.candidates[0].clone();
22780 assert!(
22781 runner
22782 .guarded_pr_message(&winner, None, true)
22783 .await
22784 .unwrap()
22785 .is_none()
22786 );
22787 let record = runner.state.github_text.as_ref().unwrap();
22788 assert!(record.rewrite.is_none());
22789 let mut old = serde_json::to_value(record).unwrap();
22791 old.as_object_mut().unwrap().remove("rewrite");
22792 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22793 assert!(old.rewrite.is_none());
22794 let id = record.question.clone().unwrap();
22795 ask::Questions::open()
22796 .update(&id, |q| {
22797 q.choices.push(crate::github_text::USE_REWRITE.into());
22798 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22799 })
22800 .unwrap();
22801 let message = runner
22802 .guarded_pr_message(&winner, None, true)
22803 .await
22804 .unwrap()
22805 .unwrap();
22806 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22807 assert_eq!(
22808 runner.state.github_text.as_ref().unwrap().choice,
22809 Some(GithubTextChoice::Fallback)
22810 );
22811 }
22812
22813 #[tokio::test]
22814 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22815 crate::run::pin_test_home();
22816 let dir = tempfile::tempdir().unwrap();
22817 let mut runner = runner_at(RunStatus::Gating);
22818 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22819 runner.state.repo = dir.path().to_owned();
22820 runner.state.candidates[0].worktree = dir.path().to_owned();
22821 seed_public_summary(&mut runner);
22822 let winner = runner.state.candidates[0].clone();
22823 assert!(
22824 runner
22825 .guarded_pr_message(&winner, None, true)
22826 .await
22827 .unwrap()
22828 .is_none()
22829 );
22830 let store = ask::Questions::open();
22831 let find = |run: &str| {
22832 store
22833 .list()
22834 .into_iter()
22835 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22836 .unwrap()
22837 };
22838 let run = runner.state.id.clone();
22839 let q = find(&run);
22840 let fp = runner
22841 .state
22842 .github_text
22843 .as_ref()
22844 .unwrap()
22845 .fingerprint
22846 .clone();
22847 let name = crate::github_text::artifact_name(&fp);
22848 let path = crate::run::artifact_path(&runner.state, &name);
22849 assert!(q.detail.contains(&path.display().to_string()));
22850 assert!(q.detail.contains("title-language: title"));
22851 let saved = std::fs::read_to_string(&path).expect("artifact written");
22852 assert!(saved.contains("日本語のタイトル"));
22853 assert!(q.cwd.is_none());
22854 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
22855 let say = |id: &str, text: &str, choice: &str| {
22857 store
22858 .update(id, |q| {
22859 q.thread.push(ask::Turn {
22860 who: ask::Who::Operator,
22861 body: text.to_owned(),
22862 at: jiff::Timestamp::now(),
22863 note: None,
22864 });
22865 q.answer(ask::Answer::Choice(choice.to_owned()))
22866 })
22867 .unwrap();
22868 };
22869 say(&q.id, "まだ日本語", "use my text");
22870 assert!(
22871 runner
22872 .guarded_pr_message(&winner, None, true)
22873 .await
22874 .unwrap()
22875 .is_none()
22876 );
22877 let second = find(&run);
22878 assert_ne!(second.id, q.id);
22879 assert!(second.detail.contains("did not pass"));
22880 assert!(second.detail.contains("日本語のタイトル"));
22883 assert!(!second.detail.contains("まだ日本語"));
22884 assert!(second.detail.contains(&path.display().to_string()));
22885 say(&second.id, "fix: retry failed requests", "use my text");
22886 let message = runner
22887 .guarded_pr_message(&winner, None, true)
22888 .await
22889 .unwrap()
22890 .expect("a vetted replacement is posted");
22891 assert_eq!(message.title, "fix: retry failed requests");
22892 assert_eq!(
22893 runner
22894 .state
22895 .github_text
22896 .as_ref()
22897 .unwrap()
22898 .chosen_title
22899 .as_deref(),
22900 Some("fix: retry failed requests")
22901 );
22902 assert!(runner.state.github_text.as_ref().unwrap().resolved);
22903 let again = runner
22905 .guarded_pr_message(&winner, None, true)
22906 .await
22907 .unwrap()
22908 .unwrap();
22909 assert_eq!(again.title, message.title);
22910 assert_eq!(
22911 store
22912 .list()
22913 .into_iter()
22914 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
22915 .count(),
22916 2
22917 );
22918 }
22919
22920 #[test]
22921 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
22922 let mut state = state_with_summary(
22926 "add retries",
22927 "TITLE: fix(web): batch reads\n- reads run.json once",
22928 );
22929 state.config.graph.language = "ja".to_owned();
22930 let m = pr_message(&state, 'A');
22931 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
22932 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
22933
22934 let task = "今回やってほしいこと: results projector を直す";
22937 let mut state = state_with_summary(task, "- no title line");
22938 state.config.graph.language = "ja".to_owned();
22939 let m = pr_message(&state, 'A');
22940 assert_eq!(
22941 m.title,
22942 format!("chore: land candidate A of run {}", state.id)
22943 );
22944 assert!(
22945 m.body.contains(&format!(
22946 "<summary>Original task</summary>\n\n{task}\n\n</details>"
22947 )),
22948 "{}",
22949 m.body
22950 );
22951 }
22952
22953 #[test]
22954 fn pr_message_scrubs_home_paths_and_addresses() {
22955 let state = state_with_summary(
22956 "fix it in /Users/someone/src/x",
22957 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
22958 );
22959 let m = pr_message(&state, 'A');
22960 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
22961 assert!(!m.body.contains(leak), "{}", m.body);
22962 }
22963 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
22964 }
22965
22966 #[test]
22967 fn pr_message_survives_a_task_that_closes_details() {
22968 let state = state_with_summary("a </details> b", "TITLE: fix: x");
22969 let m = pr_message(&state, 'A');
22970 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
22971 }
22972
22973 #[test]
22974 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
22975 let cmd = manual_merge_command(
22976 MergeStyle::Squash,
22977 Path::new("/repo"),
22978 "b",
22979 "fix: \"quoted\" $(x) `y`\n\nbody",
22980 );
22981 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
22982 }
22983
22984 #[test]
22985 fn manual_merge_command_matches_the_configured_style() {
22986 let repo = Path::new("/repo");
22987 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
22988
22989 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
22990 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
22991
22992 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
22993 assert_eq!(
22994 squash,
22995 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
22996 \"Merge magi run 0832 (candidate A)\""
22997 );
22998
22999 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23000 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23001 }
23002
23003 #[test]
23004 fn a_nudge_gets_a_quarter_of_the_budget() {
23005 assert_eq!(retry_budget(secs(1200), true), secs(300));
23007 assert_eq!(retry_budget(secs(3600), true), secs(900));
23008 }
23009
23010 #[test]
23011 fn a_resent_prompt_keeps_the_whole_budget() {
23012 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23015 assert_eq!(retry_budget(secs(60), false), secs(60));
23016 }
23017
23018 #[test]
23019 fn the_floor_never_exceeds_the_original_budget() {
23020 assert_eq!(retry_budget(secs(60), true), secs(60));
23024 assert_eq!(retry_budget(secs(480), true), secs(120));
23025 assert_eq!(retry_budget(secs(0), true), secs(0));
23026 }
23027
23028 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23029 agent::CommandEvidence {
23030 id: "item1".to_owned(),
23031 description: "cargo test".to_owned(),
23032 exit_code,
23033 result_summary: String::new(),
23034 source: "codex".to_owned(),
23035 }
23036 }
23037
23038 #[test]
23039 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23040 assert!(!has_unconfirmed_command(&[]));
23044 }
23045
23046 #[test]
23047 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23048 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23052 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23053 assert!(!has_unconfirmed_command(&[
23054 evidence(Some(0)),
23055 evidence(Some(101))
23056 ]));
23057 }
23058
23059 #[test]
23060 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23061 assert!(has_unconfirmed_command(&[
23062 evidence(Some(0)),
23063 evidence(None)
23064 ]));
23065 }
23066
23067 #[test]
23068 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23069 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23070 assert_eq!(
23071 verified_noop_claim(true, &[], text).as_deref(),
23072 Some("already fixed by b32cfc4, on main.")
23073 );
23074 }
23075
23076 #[test]
23077 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23078 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23081 assert!(verified_noop_claim(false, &[], text).is_none());
23082 }
23083
23084 #[test]
23085 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23086 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23087 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23088 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23090 }
23091
23092 #[test]
23093 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23094 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23095 }
23096
23097 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23100 runner.state.candidates = shape
23101 .iter()
23102 .enumerate()
23103 .map(|(i, &(empty, verified))| Candidate {
23104 index: i,
23105 label: (b'A' + i as u8) as char,
23106 agent: "sonnet".to_owned(),
23107 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23108 worktree: PathBuf::from(format!("/wt/{i}")),
23109 summary: String::new(),
23110 stat: String::new(),
23111 files: 0,
23112 commits: 0,
23113 empty,
23114 failed: None,
23115 verified_noop: verified.map(str::to_owned),
23116 duration_ms: 0,
23117 folded: false,
23118 })
23119 .collect();
23120 }
23121
23122 #[test]
23123 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23124 ask_test_home();
23125 let mut runner = runner_at(RunStatus::Implementing);
23126 set_candidates(
23127 &mut runner,
23128 &[
23129 (true, Some("already on main at b32cfc4")),
23130 (true, Some("same fix, see the existing test")),
23131 ],
23132 );
23133
23134 runner
23135 .after_implement()
23136 .expect("a verified no-op is not an error");
23137
23138 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23139 }
23140
23141 #[test]
23142 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23143 ask_test_home();
23144 let mut runner = runner_at(RunStatus::Implementing);
23145 set_candidates(
23149 &mut runner,
23150 &[(true, Some("already on main at b32cfc4")), (true, None)],
23151 );
23152
23153 let err = runner
23154 .after_implement()
23155 .expect_err("an unverified empty candidate must still fail the run");
23156
23157 assert!(
23158 err.to_string().contains("no candidate produced a change"),
23159 "{err}"
23160 );
23161 assert_eq!(runner.state.status, RunStatus::Failed);
23162 }
23163
23164 #[test]
23165 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23166 ask_test_home();
23167 let mut runner = runner_at(RunStatus::Implementing);
23168 set_candidates(&mut runner, &[(true, None), (true, None)]);
23169
23170 let err = runner
23171 .after_implement()
23172 .expect_err("no candidate declared anything; this is an ordinary failure");
23173
23174 assert!(
23175 err.to_string().contains("no candidate produced a change"),
23176 "{err}"
23177 );
23178 assert_eq!(runner.state.status, RunStatus::Failed);
23179 }
23180}
23181
23182#[derive(Debug)]
23185struct GraftRestoreFailed(String);
23186
23187impl std::fmt::Display for GraftRestoreFailed {
23188 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23189 write!(
23190 f,
23191 "could not restore the winner to its pre-graft commit: {}",
23192 self.0
23193 )
23194 }
23195}
23196
23197impl std::error::Error for GraftRestoreFailed {}
23198
23199#[cfg(test)]
23203mod resource_wait_tests {
23204 use super::*;
23205 use std::collections::BTreeMap;
23206
23207 fn sh(id: &str, script: &str) -> AgentSpec {
23208 AgentSpec {
23209 id: id.to_owned(),
23210 kind: crate::config::AgentKind::Command,
23211 model: None,
23212 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23213 extra_args: Vec::new(),
23214 env: BTreeMap::new(),
23215 prompt_delivery: None,
23216 max_turn_secs: None,
23217 }
23218 }
23219
23220 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23221 SeatJob {
23222 seat: SeatState::new("impl-A", &spec.id, 1),
23223 spec,
23224 cwd: dir.to_path_buf(),
23225 prompt: "do it".to_owned(),
23226 timeout,
23227 allow_write: true,
23228 sessions: false,
23229 artifacts: dir.join("art"),
23230 stem: "impl-A".to_owned(),
23231 handover: None,
23232 }
23233 }
23234
23235 fn state(resource_wait_secs: u64) -> RunState {
23236 let mut config = Config::default();
23237 config.graph.timeout_resource_wait = resource_wait_secs;
23238 RunState::new(
23239 PathBuf::from("/repo"),
23240 "main".to_owned(),
23241 "abc1234".to_owned(),
23242 "task".to_owned(),
23243 config,
23244 )
23245 }
23246
23247 fn hold(cache: &Path) -> crate::cache::Guard {
23248 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23249 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23250 crate::cache::AcquireOutcome::Acquired(g) => g,
23251 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23252 }
23253 }
23254
23255 async fn run(
23256 jobs: Vec<SeatJob>,
23257 sem: Arc<Semaphore>,
23258 cache: Option<&Path>,
23259 st: &mut RunState,
23260 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23261 let prompts = Prompts::default();
23262 let ctx = WaveCtx {
23263 run: "r",
23264 node: "implement",
23265 prompts: &prompts,
23266 cache,
23267 round: None,
23268 carry_seats: false,
23269 pauses: None,
23270 };
23271 wave(jobs, sem, &ctx, st, 0).await
23272 }
23273
23274 #[tokio::test]
23275 async fn verification_cache_wait_preserves_the_command_budget() {
23276 crate::run::pin_test_home();
23277 let tmp = tempfile::tempdir().unwrap();
23278 let cache = tmp.path().join("target");
23279 let guard = hold(&cache);
23280 let release = tokio::spawn(async move {
23281 tokio::time::sleep(Duration::from_millis(100)).await;
23282 guard.release();
23283 });
23284 let mut st = state(10);
23285 let budget = Duration::from_millis(50);
23286 let wait = repository_wait(&st, None);
23287 let (outcomes, _) = with_cache_lease(
23288 &mut st,
23289 Some(&cache),
23290 "gate",
23291 "gate",
23292 tmp.path(),
23293 "abc1234",
23294 budget,
23295 "test gate",
23296 &wait,
23297 |_, available| async move {
23298 assert_eq!(
23299 available, budget,
23300 "contention must not shorten verification"
23301 );
23302 (Vec::new(), false, Vec::new())
23303 },
23304 )
23305 .await
23306 .unwrap();
23307 release.await.unwrap();
23308 assert!(outcomes.is_empty(), "{outcomes:?}");
23309 assert!(
23310 st.events
23311 .iter()
23312 .any(|event| event.message.contains("waiting for"))
23313 );
23314 assert!(st.resource_wait.is_none());
23315 hold(&cache).release();
23317 }
23318
23319 #[tokio::test]
23320 async fn verification_cache_wait_expires_without_starting_commands() {
23321 crate::run::pin_test_home();
23322 let tmp = tempfile::tempdir().unwrap();
23323 let cache = tmp.path().join("target");
23324 let _held = hold(&cache);
23325 let mut st = state(0);
23326 let wait = repository_wait(&st, None);
23327 let (outcomes, _) = with_cache_lease(
23328 &mut st,
23329 Some(&cache),
23330 "gate",
23331 "gate",
23332 tmp.path(),
23333 "abc1234",
23334 Duration::from_secs(60),
23335 "test gate",
23336 &wait,
23337 |_, _| async { panic!("commands must not start without the cache lease") },
23338 )
23339 .await
23340 .unwrap();
23341 assert_eq!(outcomes.len(), 1);
23342 assert!(outcomes[0].resource_blocked);
23343 assert!(st.resource_wait.is_none());
23344 assert!(matches!(
23345 st.resource_block.unwrap().cause,
23346 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23347 ));
23348 }
23349
23350 #[tokio::test]
23351 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23352 crate::run::pin_test_home();
23353 let tmp = tempfile::tempdir().unwrap();
23354 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23355 let cache = tmp.path().join("cache");
23356 let mut state = state(21600);
23357 let wait = repository_wait(&state, None);
23358 let (outcomes, _) = with_cache_lease(
23359 &mut state,
23360 Some(&cache),
23361 "gate",
23362 "gate",
23363 tmp.path(),
23364 "head",
23365 Duration::from_secs(60),
23366 "gate",
23367 &wait,
23368 |_, _| async { panic!("freshness failure must not start verification") },
23369 )
23370 .await
23371 .unwrap();
23372 assert_eq!(outcomes.len(), 1);
23373 assert!(outcomes[0].resource_blocked);
23374 let block = state.resource_block.unwrap();
23375 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23376 assert!(!block.description().contains("timed out"));
23377 }
23378
23379 #[tokio::test]
23380 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23381 crate::run::pin_test_home();
23382 let tmp = tempfile::tempdir().unwrap();
23383 let cache = tmp.path().join("cache");
23384 let _held = hold(&cache);
23385 let mut state = state(21600);
23386 let pause = Pause::new();
23387 let interrupt = Pause::new();
23388 let cause = ParkCause::Operator;
23389 pause.park_for(cause.clone());
23390 let prompts = Prompts::default();
23391 let ctx = WaveCtx {
23392 run: "test",
23393 node: "fix",
23394 prompts: &prompts,
23395 cache: Some(&cache),
23396 round: None,
23397 carry_seats: false,
23398 pauses: Some((&pause, &interrupt)),
23399 };
23400 let result = wave(
23401 vec![job(
23402 sh("writer", "exit 99"),
23403 tmp.path(),
23404 Duration::from_secs(1),
23405 )],
23406 Arc::new(Semaphore::new(1)),
23407 &ctx,
23408 &mut state,
23409 0,
23410 )
23411 .await;
23412 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23413 assert_eq!(result[0].1.turns, 0);
23414 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23415 assert_eq!(
23416 state.resource_block.unwrap().cause,
23417 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23418 );
23419 assert!(state.active.is_empty());
23420 }
23421
23422 #[tokio::test]
23423 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23424 crate::run::pin_test_home();
23425 let tmp = tempfile::tempdir().unwrap();
23426 let cache = tmp.path().join("target-a");
23427 let guard = hold(&cache);
23428 tokio::spawn(async move {
23430 tokio::time::sleep(Duration::from_millis(1500)).await;
23431 guard.release();
23432 });
23433 let mut st = state(60);
23434 let results = run(
23435 vec![job(
23436 sh("alpha", "echo ok"),
23437 tmp.path(),
23438 Duration::from_secs(1),
23439 )],
23440 Arc::new(Semaphore::new(1)),
23441 Some(&cache),
23442 &mut st,
23443 )
23444 .await;
23445 assert!(
23446 matches!(results[0].2, AgentOutcome::Ok(_)),
23447 "the wait must not be charged to the agent"
23448 );
23449 assert!(st.resource_block.is_none());
23450 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23451 assert!(st.resource_wait.is_none() && st.active.is_empty());
23452 }
23453
23454 #[tokio::test]
23455 #[cfg(unix)]
23456 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23457 use std::os::fd::AsRawFd as _;
23458 crate::run::pin_test_home();
23459 let tmp = tempfile::tempdir().unwrap();
23460 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23461 let common = tmp.path().join(".git");
23462 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23463 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23464 assert_eq!(
23465 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23466 0
23467 );
23468 let mut st = state(1);
23469 st.config.coordination = crate::config::test_coordination();
23470 let mut reader = job(
23471 sh("reader", "echo must-not-run"),
23472 tmp.path(),
23473 Duration::from_secs(30),
23474 );
23475 reader.allow_write = false;
23476 let results = run(
23477 vec![reader.clone()],
23478 Arc::new(Semaphore::new(1)),
23479 None,
23480 &mut st,
23481 )
23482 .await;
23483 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23484 assert_eq!(results[0].1.turns, 0);
23485 assert!(FailClass::of(&results[0].2).is_none());
23486 assert_eq!(
23487 st.resource_block.as_ref().unwrap().kind,
23488 "repository coordination"
23489 );
23490 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23491 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23492 drop(external);
23493 let editor = crate::repo_lock::SeatLease::for_checkout(
23494 tmp.path(),
23495 true,
23496 crate::config::test_coordination().as_ref(),
23497 )
23498 .await
23499 .unwrap();
23500 let pause = Pause::new();
23501 let interrupt = Pause::new();
23502 let prompts = Prompts::default();
23503 let ctx = WaveCtx {
23504 run: "r",
23505 node: "review",
23506 prompts: &prompts,
23507 cache: None,
23508 round: None,
23509 carry_seats: false,
23510 pauses: Some((&pause, &interrupt)),
23511 };
23512 let (results, ()) = tokio::join!(
23513 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23514 async {
23515 tokio::time::sleep(Duration::from_millis(50)).await;
23516 pause.park();
23517 }
23518 );
23519 assert!(
23520 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23521 );
23522 assert_eq!(results[0].1.turns, 0);
23523 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23524 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23525 drop(editor);
23526 }
23527
23528 #[tokio::test]
23529 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23530 crate::run::pin_test_home();
23531 let tmp = tempfile::tempdir().unwrap();
23532 let cache = tmp.path().join("target-b");
23533 let _held = hold(&cache);
23534 let mut st = state(1);
23535 let results = run(
23536 vec![job(
23537 sh("alpha", "echo ok"),
23538 tmp.path(),
23539 Duration::from_secs(1),
23540 )],
23541 Arc::new(Semaphore::new(1)),
23542 Some(&cache),
23543 &mut st,
23544 )
23545 .await;
23546 let (_, seat, out) = &results[0];
23547 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23548 assert!(wave_blocked(&results));
23549 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23550 assert_eq!(seat.turns, 0, "nothing was asked");
23551 assert!(
23552 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23553 "no agent was spawned"
23554 );
23555 assert!(FailClass::of(out).is_none(), "not a failure class");
23556 let block = st.resource_block.as_ref().expect("recorded for the resume");
23557 assert_eq!(block.kind, "shared build cache");
23558 assert!(matches!(
23559 block.cause,
23560 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23561 ));
23562 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23563 assert!(st.active.is_empty());
23564 }
23565
23566 #[tokio::test]
23567 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23568 crate::run::pin_test_home();
23569 let tmp = tempfile::tempdir().unwrap();
23570 let mut st = state(1);
23571 let results = run(
23572 vec![job(
23573 sh("alpha", "echo ok"),
23574 tmp.path(),
23575 Duration::from_secs(1),
23576 )],
23577 Arc::new(Semaphore::new(0)),
23578 None,
23579 &mut st,
23580 )
23581 .await;
23582 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23583 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23584 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23585 }
23586
23587 #[tokio::test]
23588 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23589 crate::run::pin_test_home();
23590 let tmp = tempfile::tempdir().unwrap();
23591 let mut st = state(60);
23592 let results = run(
23593 vec![job(
23594 sh("alpha", "sleep 30"),
23595 tmp.path(),
23596 Duration::from_secs(1),
23597 )],
23598 Arc::new(Semaphore::new(1)),
23599 None,
23600 &mut st,
23601 )
23602 .await;
23603 let out = &results[0].2;
23604 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23605 let class = FailClass::of(out).expect("a failure");
23606 assert_eq!(class, FailClass::Timeout);
23607 assert!(should_hand_over(None, &class), "existing handover policy");
23608 assert!(st.resource_block.is_none());
23609 }
23610}