1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 if state.startup.as_ref().is_some_and(|s| !s.complete)
1470 || (state.status == RunStatus::Prep
1471 && state.instruction.trim().is_empty()
1472 && state.candidates.is_empty()
1473 && state.prep_complete == Some(false))
1474 {
1475 bail!(
1476 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477 );
1478 }
1479 crate::pr_head::ensure_owner(&state)?;
1480 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481 bail!(
1482 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483 );
1484 }
1485 if let Some(to) = &state.released_to {
1486 bail!(
1487 "run {} cannot be resumed: its worktree was released to run {}",
1488 state.short(),
1489 crate::run::short_of(to)
1490 );
1491 }
1492 let roles = state.config.resolve_roles()?;
1493 if let Some(id) = &state.config.graph.selected_implementer
1494 && state.candidates.iter().any(|c| c.agent != *id)
1495 {
1496 bail!(
1497 "replacement implementer pin does not match the saved candidates; task must stay held"
1498 );
1499 }
1500 let max_parallel = state.config.graph.max_parallel.max(1);
1501 Ok(Self {
1502 state,
1503 roles,
1504 sem: Arc::new(Semaphore::new(max_parallel)),
1505 pause: Pause::new(),
1506 interrupt: Pause::new(),
1507 })
1508 }
1509
1510 pub async fn execute(&mut self) -> Result<()> {
1517 anyhow::ensure!(
1518 self.state.startup.as_ref().is_none_or(|s| s.complete),
1519 "startup incomplete; inspect retained evidence before explicit recovery"
1520 );
1521 if let Some(selection) = &self.state.pr_recovery
1522 && (selection.source_proof.is_some() || selection.source_run.is_some())
1523 {
1524 Box::pin(selection.verify_source(&self.state.repo)).await?;
1525 }
1526 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529 || self.state.land_approval.is_some()
1530 || self.state.merge_decision.is_some()
1531 || !self.state.merge_repairs.is_empty()
1532 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533 {
1534 let claim = FixClaim::acquire(&self.state.dir())?;
1535 let disk = RunState::load(&self.state.id)?;
1536 let checkpoint = |s: &RunState| {
1537 (
1538 s.merge_repairs.len(),
1539 s.merge_repairs
1540 .last()
1541 .map(|r| (r.question.clone(), r.phase)),
1542 s.land_approval
1543 .as_ref()
1544 .map(|a| (a.question.clone(), a.head.clone())),
1545 )
1546 };
1547 if checkpoint(&disk) != checkpoint(&self.state) {
1548 bail!(
1549 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550 );
1551 }
1552 Some(claim)
1553 } else {
1554 None
1555 };
1556 let mut execution = self.claim_driver().await?;
1557 let runner = &mut *execution.runner;
1558 let wait = runner.repository_wait();
1559 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562 let result = match result {
1563 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564 record_repository_block(&mut runner.state, &e);
1565 runner.park_here()?;
1566 Ok(())
1567 }
1568 other => other,
1569 };
1570 execution.driver.finish(&mut runner.state).await?;
1571 let ended = if result.is_err() {
1572 Some({
1573 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574 if let Err(error) = &result {
1575 notice.message = format!(
1576 "Run {} stopped with an error: {}",
1577 runner.state.id,
1578 error.to_string().lines().next().unwrap_or("unknown error")
1579 );
1580 }
1581 notice
1582 })
1583 } else {
1584 crate::notices::run_ended(&runner.state)
1585 };
1586 if let Some(notice) = ended {
1587 crate::notices::raise(notice);
1588 }
1589 result
1590 }
1591
1592 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594 }
1595
1596 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597 let wait = self.repository_wait();
1598 coordinate_operation(&mut self.state, cwd, &wait).await
1599 }
1600
1601 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603 let owner = match crate::pr_head::owner_lock_wait(
1604 &crate::run::home(),
1605 &self.state.id,
1606 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607 )
1608 .await
1609 {
1610 Ok(owner) => owner,
1611 Err(error) => {
1612 let detail = format!("worktree maintenance or ownership update: {error:#}");
1613 self.state.resource_block = Some(crate::run::ResourceBlock {
1617 cause: crate::run::ResourceBlockCause::Other,
1618 node: "start".into(),
1619 kind: "worktree maintenance".into(),
1620 detail: detail.clone(),
1621 at: Timestamp::now(),
1622 });
1623 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624 }
1625 };
1626 let disk = RunState::load(&self.state.id)?;
1627 crate::pr_head::ensure_owner(&disk)?;
1628 let claimed =
1629 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630 anyhow::ensure!(
1635 disk.liveness(claimed) != crate::run::Liveness::Live,
1636 "run already has a live driver; wait for its execution to end"
1637 );
1638 if let Some(to) = &disk.released_to {
1639 bail!(
1640 "run {} cannot continue: its worktree was released to run {}",
1641 self.state.short(),
1642 crate::run::short_of(to)
1643 );
1644 }
1645 anyhow::ensure!(
1646 serde_json::to_value(&disk.base_sync_recovery)?
1647 == serde_json::to_value(&self.state.base_sync_recovery)?,
1648 "base recovery changed since this run was loaded; reload the saved run"
1649 );
1650 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651 let mut execution = DriverExecution {
1652 runner: self,
1653 driver,
1654 };
1655 let _owner = owner;
1658 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659 .await?;
1660 let state = &mut execution.runner.state;
1661 state.parked = false;
1666 state.resource_wait = None;
1667 state.resource_block = None;
1668 state.clear_active();
1675 let pid = std::process::id();
1690 state.driver_pid = Some(pid);
1691 state.driver_started_at = crate::proc::process_started_at(pid);
1692 state.driver_exited = false;
1693 state.driver_claimed = Some(true);
1694 state.begin_pass();
1695 execution.driver.bind(state);
1696 state.save()?;
1697 execution.driver.record(state)?;
1698 Ok(execution)
1699 }
1700
1701 async fn execute_granted_repair(&mut self) -> Result<()> {
1704 use crate::repair_budget::{Kind, Phase};
1705 let index = self.state.repair_grants.len() - 1;
1706 let winner = self
1707 .state
1708 .winner()
1709 .cloned()
1710 .context("repair winner missing")?;
1711 loop {
1712 let grant = self.state.repair_grants[index].clone();
1713 match grant.phase {
1714 Phase::Pending => {
1715 if grant.spent >= grant.receipt.request.rounds {
1716 self.state.repair_grants[index].phase = Phase::Complete;
1717 self.state.status = RunStatus::Blocked;
1718 self.state.save()?;
1719 return Ok(());
1720 }
1721 self.state.repair_grants[index].spent += 1;
1724 self.state.repair_grants[index].phase = Phase::Fixing;
1725 self.state.save()?;
1726 match grant.receipt.request.kind {
1727 Kind::Gate => {
1728 let failed = self.state.gate.clone();
1729 let _ = self.gate_fix_round(&winner, &failed).await?;
1730 }
1731 Kind::Review => {
1732 let mut record = self
1733 .state
1734 .reviews
1735 .last()
1736 .cloned()
1737 .context("review failure missing")?;
1738 let round = record.round;
1739 let step = self.run_fix_round(&winner, &mut record).await?;
1740 self.state.reviews[round - 1] = record;
1741 if matches!(step, FixStep::Recorded { report_lost: true }) {
1742 self.state.repair_grants[index].phase = Phase::Complete;
1743 self.state.status = RunStatus::Blocked;
1744 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745 self.state.save()?;
1746 return Ok(());
1747 }
1748 if matches!(step, FixStep::Resource) {
1749 self.stop_for_resource()?;
1750 return Ok(());
1751 }
1752 self.state.save()?;
1753 }
1754 }
1755 if self.resource_halt()? {
1756 return Ok(());
1757 }
1758 }
1759 Phase::Fixing => {
1760 let _coordination = self.coordinate(&winner.worktree).await?;
1763 let rescued = git::rescue_commit(
1764 &winner.worktree,
1765 "magi: preserve interrupted granted repair",
1766 )
1767 .await?;
1768 self.state.note_withheld("repair-grant", &rescued.withheld);
1769 self.state
1772 .verification_history
1773 .push(crate::run::VerificationEvidence {
1774 superseded_by: None,
1775 archived_at: Some(Timestamp::now()),
1776 fixes_spent: Some(self.state.gate_fixes.len()),
1777 gate_head: self.state.gate_head.take(),
1778 gate_ran: std::mem::take(&mut self.state.gate_ran),
1779 gate: std::mem::take(&mut self.state.gate),
1780 e2e: self.state.head_e2e.take(),
1781 });
1782 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783 self.state
1784 .verification_history
1785 .push(crate::run::VerificationEvidence {
1786 superseded_by: None,
1787 archived_at: Some(Timestamp::now()),
1788 fixes_spent: Some(self.state.gate_fixes.len()),
1789 gate_head: None,
1790 gate_ran: false,
1791 gate: Vec::new(),
1792 e2e: Some(e2e),
1793 });
1794 }
1795 self.state.repair_grants[index].review_start = self.state.reviews.len();
1796 self.state.repair_grants[index].head =
1797 git::rev_parse(&winner.worktree, "HEAD").await?;
1798 self.state.repair_grants[index].phase = Phase::Reviewing;
1799 self.state.save()?;
1800 }
1801 Phase::Reviewing => {
1802 self.review_loop_pass().await?;
1803 if self.resource_halt()? {
1804 return Ok(());
1805 }
1806 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807 let clean = self.state.reviews.len() > grant.review_start
1808 && self.state.reviews.last().is_some_and(|r| {
1809 r.head == head
1810 && r.clean
1811 && r.expected > 0
1812 && r.answered == r.expected
1813 && matches!(
1814 r.e2e_status(),
1815 E2eStatus::Passed | E2eStatus::NotConfigured
1816 )
1817 });
1818 if !clean {
1819 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821 == Kind::Review
1822 && grant.spent < grant.receipt.request.rounds
1823 && self.state.reviews.last().is_some_and(|r| {
1824 r.fix.is_none()
1825 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826 }) {
1827 Phase::Pending
1828 } else {
1829 Phase::Complete
1830 };
1831 self.state.status = RunStatus::Blocked;
1832 self.state.event(
1833 "repair-grant",
1834 "fresh review did not clear the repaired head; no publication",
1835 );
1836 self.state.save()?;
1837 if self.state.repair_grants[index].phase == Phase::Pending {
1838 continue;
1839 }
1840 return Ok(());
1841 }
1842 self.state.repair_grants[index].phase = Phase::Gating;
1843 self.state.save()?;
1844 }
1845 Phase::Gating => {
1846 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847 self.state.reconcile_verification_head(&head);
1848 let wait = self.repository_wait();
1849 if self
1850 .state
1851 .reviews
1852 .last()
1853 .is_none_or(|r| r.head != head || !r.clean)
1854 {
1855 self.state.status = RunStatus::Blocked;
1856 self.state.repair_grants[index].phase = Phase::Complete;
1857 self.state.event(
1858 "repair-grant",
1859 "head changed since fresh review; repair held",
1860 );
1861 self.state.save()?;
1862 return Ok(());
1863 }
1864 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865 return Ok(());
1866 }
1867 let outcomes = self.run_gate(&winner).await?;
1868 if verify_inconclusive(&outcomes) {
1869 return Ok(());
1870 }
1871 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872 if after != head {
1873 self.state.repair_grants[index].review_start = self.state.reviews.len();
1874 self.state.repair_grants[index].head = after;
1875 self.state.repair_grants[index].phase = Phase::Reviewing;
1876 self.state.status = RunStatus::Blocked;
1877 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878 self.state.save()?;
1879 return Ok(());
1880 }
1881 let passed = outcomes.iter().all(CommandOutcome::ok);
1882 self.state.repair_grants[index].succeeded = passed;
1883 self.state.gate = outcomes;
1884 self.state.gate_ran = true;
1885 self.state.status = if passed {
1886 RunStatus::Gating
1887 } else {
1888 RunStatus::Blocked
1889 };
1890 self.state.repair_grants[index].phase = if !passed
1891 && grant.receipt.request.kind == Kind::Gate
1892 && grant.spent < grant.receipt.request.rounds
1893 {
1894 Phase::Pending
1895 } else {
1896 Phase::Complete
1897 };
1898 if passed && self.state.config.merge.mode == MergeMode::Pr {
1899 self.state.pr_publication_pending = true;
1901 } else if passed {
1902 self.state.status = RunStatus::Ready;
1903 }
1904 self.state.save()?;
1905 if !passed {
1906 continue;
1907 }
1908 if self.state.config.merge.mode != MergeMode::Pr {
1912 self.state.status = RunStatus::Ready;
1913 self.state.event(
1914 "repair-grant",
1915 "repair verified locally; grant carries no merge authorization",
1916 );
1917 self.state.save()?;
1918 return Ok(());
1919 }
1920 return self.merge().await;
1921 }
1922 Phase::Complete => return Ok(()),
1923 }
1924 }
1925 }
1926
1927 async fn execute_graph(&mut self) -> Result<()> {
1928 self.state.configure_recovery_resume()?;
1929 if self
1930 .state
1931 .repair_grants
1932 .last()
1933 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1934 {
1935 return self.execute_granted_repair().await;
1936 }
1937 if self
1938 .state
1939 .repair_grants
1940 .last()
1941 .is_some_and(|g| !g.succeeded)
1942 {
1943 self.state.status = RunStatus::Blocked;
1944 self.state.save()?;
1945 return Ok(());
1946 }
1947 self.persist_candidates().await?;
1950 if self.state.base_sync_blocked() {
1953 self.sync_to_base().await?;
1954 if !self.state.base_sync_blocked()
1955 && self.state.status != RunStatus::AlreadyInBase
1956 && !self.resource_halt()?
1957 {
1958 self.finish_after_tally().await?;
1959 }
1960 return Ok(());
1961 }
1962 if self
1963 .state
1964 .merge_decision
1965 .as_ref()
1966 .is_some_and(|decision| !decision.closed)
1967 && self.state.status != RunStatus::Landing
1968 {
1969 if crate::merge_decision::resumable_hold(&self.state) {
1970 self.state.merge = Some(MergeOutcome {
1971 mode: MergeMode::Pr,
1972 ok: false,
1973 detail: self
1974 .state
1975 .merge_decision
1976 .as_ref()
1977 .unwrap()
1978 .target
1979 .pr
1980 .clone(),
1981 empty: false,
1982 });
1983 self.run_land().await?;
1984 self.settle_questions();
1985 return Ok(());
1986 }
1987 bail!(
1988 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1989 );
1990 }
1991 if self.state.config.graph.selected_implementer.is_some()
1992 && self.state.status == RunStatus::Blocked
1993 && self.state.tally.is_none()
1994 && self.state.candidate_preservation.is_empty()
1995 && self.state.interrupted_implementations.is_empty()
1996 {
1997 bail!(
1998 "selected implementation is held; explicitly release the owning task to retry it"
1999 );
2000 }
2001
2002 if self.state.merge_repairs.last().is_some_and(|r| {
2003 matches!(
2004 r.phase,
2005 crate::run::MergeRepairPhase::Requested
2006 | crate::run::MergeRepairPhase::Fixing
2007 | crate::run::MergeRepairPhase::Reviewing
2008 )
2009 }) {
2010 if self.repair_merge_review().await? {
2011 self.run_land().await?;
2012 }
2013 self.settle_questions();
2014 return Ok(());
2015 }
2016 if self.state.pr_publication_pending {
2019 self.merge().await?;
2020 if self.resource_halt()? {
2021 return Ok(());
2022 }
2023 self.state.save()?;
2024 self.settle_questions();
2025 return Ok(());
2026 }
2027 if self.state.status == RunStatus::Stalled {
2040 if self.recover_stall().await? {
2041 self.finish_after_tally().await?;
2042 } else {
2043 self.state.save()?;
2045 }
2046 return Ok(());
2047 }
2048 if self.state.status == RunStatus::Landing
2058 || crate::handover::landing_checkpoint(&self.state)
2059 {
2060 self.state.land_sync_pending = true;
2061 self.state.save()?;
2062 self.run_land().await?;
2063 self.settle_questions();
2068 return Ok(());
2069 }
2070 if self.state.github_text.is_some()
2075 && self.state.merge.is_none()
2076 && self.state.status == RunStatus::Gating
2077 {
2078 self.merge().await?;
2079 if self.resource_halt()? {
2080 return Ok(());
2081 }
2082 self.state.save()?;
2083 return Ok(());
2084 }
2085 if let Err(e) = self.prep().await {
2086 if e.is::<crate::repo_lock::ResourceBlocked>() {
2087 return Err(e);
2088 }
2089 self.state.status = RunStatus::Blocked;
2090 self.state
2091 .prep_failure
2092 .get_or_insert_with(|| format!("{e:#}"));
2093 self.state
2094 .event("prep", format!("preparation stopped: {e:#}"));
2095 self.state.save()?;
2096 return Err(e);
2097 }
2098 if self.park_here()? {
2099 return Ok(());
2100 }
2101 self.advise().await?;
2102 if self.park_here()? {
2103 return Ok(());
2104 }
2105 self.implement().await?;
2106 if self.park_here()? {
2107 return Ok(());
2108 }
2109 if self.state.status == RunStatus::VerifiedNoop {
2113 return Ok(());
2114 }
2115 self.judge().await?;
2116 if self.park_here()? {
2117 return Ok(());
2118 }
2119 self.deliberate().await?;
2120 if self.park_here()? {
2121 return Ok(());
2122 }
2123 self.vote().await?;
2124 if self.park_here()? {
2125 return Ok(());
2126 }
2127 self.tally()?;
2128 if self.state.status == RunStatus::Stalled {
2133 self.state.save()?;
2137 return Ok(());
2138 }
2139 self.finish_after_tally().await?;
2140 Ok(())
2141 }
2142
2143 fn stop_for_resource(&mut self) -> Result<()> {
2148 if self.pause.parked() || self.interrupt.parked() {
2151 self.park_here()?;
2152 return Ok(());
2153 }
2154 if self.state.resource_block.is_some() {
2155 self.state.status = RunStatus::Blocked;
2156 }
2157 self.state.save()
2158 }
2159
2160 fn resource_halt(&mut self) -> Result<bool> {
2162 if self.state.resource_block.is_none() {
2163 return Ok(false);
2164 }
2165 self.stop_for_resource()?;
2166 Ok(true)
2167 }
2168
2169 fn park_here(&mut self) -> Result<bool> {
2176 let parking = self.pause.parked() || self.interrupt.parked();
2177 if !parking && self.state.resource_block.is_some() {
2178 self.stop_for_resource()?;
2179 return Ok(true);
2180 }
2181 if !self.pause.parked() && !self.interrupt.parked() {
2186 return Ok(false);
2187 }
2188 if self.state.parked {
2189 return Ok(true);
2190 }
2191 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2192 if let Some(block) = &mut self.state.resource_block
2193 && matches!(
2194 block.cause,
2195 crate::run::ResourceBlockCause::Cancelled { .. }
2196 )
2197 {
2198 block.cause = crate::run::ResourceBlockCause::Cancelled {
2199 park: cause.clone(),
2200 };
2201 }
2202 let why = match cause.as_ref().map(ParkCause::label) {
2203 Some(reason) => format!(
2204 "parked after `{}` ({reason}) — resume to carry on from here",
2205 self.state.status.as_str()
2206 ),
2207 None => format!(
2208 "parked after `{}` — resume to carry on from here",
2209 self.state.status.as_str()
2210 ),
2211 };
2212 self.state.event("park", why);
2213 self.state
2214 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2215 self.state.parked = true;
2216 self.state.save()?;
2217 Ok(true)
2218 }
2219
2220 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2222 if self.state.prep_failure.is_none() {
2223 return Ok(());
2224 }
2225 let failed_at = self
2226 .state
2227 .events
2228 .iter()
2229 .rev()
2230 .find(|event| event.node == "prep")
2231 .map_or(self.state.updated_at, |event| event.at);
2232 anyhow::ensure!(
2233 approval_at > failed_at,
2234 "preparation retry requires a new explicit release after the failure"
2235 );
2236 self.state.prep_failure = None;
2237 self.state.prep_complete = Some(false);
2238 self.state
2239 .event("prep-retry", "operator approved retry of preparation");
2240 Ok(())
2241 }
2242
2243 pub fn on_pause(&mut self, pause: Pause) {
2245 self.pause = pause;
2246 }
2247
2248 pub fn watch_interrupt(&mut self, pause: Pause) {
2254 self.interrupt = pause;
2255 }
2256
2257 fn settle_questions(&mut self) {
2277 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2278 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2279 }
2280 }
2281
2282 async fn finish_after_tally(&mut self) -> Result<()> {
2285 self.graft().await?;
2288 if self.resource_halt()? {
2289 return Ok(());
2290 }
2291 self.fold_losers().await?;
2292 self.sync_to_base().await?;
2297 if self.hold_base_sync().await? || self.resource_halt()? {
2298 return Ok(());
2299 }
2300 if self.state.status == RunStatus::AlreadyInBase {
2301 return Ok(());
2302 }
2303 self.review_loop().await?;
2304 if self.resource_halt()? {
2305 return Ok(());
2306 }
2307 self.sync_to_base().await?;
2308 if self.hold_base_sync().await? || self.resource_halt()? {
2309 return Ok(());
2310 }
2311 if self.state.status == RunStatus::AlreadyInBase {
2312 return Ok(());
2313 }
2314 self.gate().await?;
2315 self.merge().await?;
2316 if self.resource_halt()? {
2317 return Ok(());
2318 }
2319 self.state.save()?;
2320 Ok(())
2321 }
2322
2323 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2330 let result = match ensure_setup_config(&mut self.state, repo).await {
2331 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2332 Err(e) => Err(e),
2333 };
2334 if let Err(e) = result {
2335 git::worktree_remove_clean(repo, wt).await.ok();
2336 if in_prep {
2337 self.rollback_prep(repo).await;
2338 }
2339 self.state.status = RunStatus::Blocked;
2340 self.state.event(
2341 "setup",
2342 format!("worktree setup failed in {}: {e:#}", wt.display()),
2343 );
2344 self.state.save()?;
2345 return Err(e);
2346 }
2347 Ok(())
2348 }
2349
2350 async fn rollback_prep(&mut self, repo: &Path) {
2352 let root = self.state.worktree_root();
2353 for c in std::mem::take(&mut self.state.candidates) {
2354 if matches!(
2355 git::worktree_remove_clean(repo, &c.worktree).await,
2356 Ok(true)
2357 ) {
2358 git::branch_delete(repo, &c.branch).await.ok();
2359 } else {
2360 self.state.candidates.push(c);
2361 }
2362 }
2363 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2364 for j in 1..=seats {
2365 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2366 .await
2367 .ok();
2368 }
2369 for k in 1..=self.state.config.graph.advisors {
2370 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2371 .await
2372 .ok();
2373 }
2374 }
2375
2376 async fn prep(&mut self) -> Result<()> {
2377 if let Some(reason) = &self.state.prep_failure {
2378 bail!(
2379 "candidate preparation previously failed: {reason}; correct the source before retrying"
2380 );
2381 }
2382 if self
2383 .state
2384 .prep_complete
2385 .unwrap_or(!self.state.candidates.is_empty())
2386 {
2387 return Ok(());
2388 }
2389 let repo = self.state.repo.clone();
2390 let _coordination = self.coordinate(&repo).await?;
2393 let base = self.state.base_commit.clone();
2394 let plan = self.state.sources.plan(&repo, &base).await?;
2395 self.rollback_prep(&repo).await;
2396 if !self.state.candidates.is_empty() {
2397 bail!(
2398 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2399 );
2400 }
2401 self.state.prep_complete = Some(false);
2402 self.state.status = RunStatus::Prep;
2403 self.state.save()?;
2404 let repo = self.state.repo.clone();
2405 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2406 let root = self.state.worktree_root();
2407 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2408
2409 let hooks_dir = self.state.dir().join("hooks");
2412 if self.state.config.blind.commit_msg_hook {
2413 std::fs::create_dir_all(&hooks_dir)
2414 .with_context(|| format!("create {}", hooks_dir.display()))?;
2415 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2416 let path = hooks_dir.join("commit-msg");
2417 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2418 make_executable(&path)?;
2419 git::acquire_worktree_config(&repo).await?;
2427 self.state.enabled_worktree_config = true;
2428 }
2429
2430 ensure_setup_config(&mut self.state, &repo).await?;
2433
2434 for (index, (spec, label)) in self
2435 .roles
2436 .implementers
2437 .clone()
2438 .into_iter()
2439 .zip(labels)
2440 .enumerate()
2441 {
2442 let branch = self.state.branch_for(label);
2443 let worktree = root.join(format!("cand-{label}"));
2444 self.state.event(
2448 "prep-intent",
2449 serde_json::json!({
2450 "branch": branch,
2451 "worktree": worktree,
2452 "start": start,
2453 "picks": plan.picks,
2454 })
2455 .to_string(),
2456 );
2457 self.state.save()?;
2458 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2459 if self.state.config.blind.commit_msg_hook {
2460 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2461 }
2462 git::local_exclude(&worktree, "/.magi/").await?;
2463 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2464 git::worktree_remove_clean(&repo, &worktree).await.ok();
2467 git::branch_delete(&repo, &branch).await.ok();
2468 self.rollback_prep(&repo).await;
2469 self.state.status = RunStatus::Blocked;
2470 self.state
2471 .event("prep", format!("worktree setup failed: {e:#}"));
2472 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2473 self.state.save()?;
2474 return Err(e);
2475 }
2476 self.state.candidates.push(Candidate {
2477 index,
2478 label,
2479 agent: spec.id.clone(),
2480 branch,
2481 worktree,
2482 summary: String::new(),
2483 stat: String::new(),
2484 files: 0,
2485 commits: 0,
2486 empty: false,
2487 failed: None,
2488 verified_noop: None,
2489 duration_ms: 0,
2490 folded: false,
2491 });
2492 self.state.save()?;
2493 for pick in &plan.picks {
2494 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2495 {
2496 self.state.status = RunStatus::Blocked;
2497 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2498 self.state
2499 .event("prep", format!("cannot apply referenced commit: {e}"));
2500 self.state.save()?;
2501 return Err(e);
2502 }
2503 }
2504 }
2505
2506 for j in 1..=self.roles.judges.len() {
2507 let wt = root.join(format!("judge-{j}"));
2508 if !wt.exists() {
2509 git::worktree_add_detached(&repo, &wt, &base).await?;
2510 self.setup_seat_worktree(&repo, &wt, true).await?;
2511 }
2512 }
2513
2514 if self.state.config.graph.advise {
2522 for k in 1..=self.state.config.graph.advisors {
2523 let wt = root.join(format!("advisor-{k}"));
2524 if !wt.exists() {
2525 git::worktree_add_detached(&repo, &wt, &base).await?;
2526 self.setup_seat_worktree(&repo, &wt, true).await?;
2527 }
2528 }
2529 }
2530
2531 let authors: Vec<&str> = self
2536 .roles
2537 .implementers
2538 .iter()
2539 .map(|a| a.id.as_str())
2540 .collect();
2541 let overlap: Vec<String> = self
2542 .roles
2543 .judges
2544 .iter()
2545 .enumerate()
2546 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2547 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2548 .collect();
2549 if !overlap.is_empty() {
2550 let note = format!(
2551 "{} also authored a candidate; blind, but the panel is less \
2552 independent than {} distinct agents would be",
2553 overlap.join(", "),
2554 self.roles.judges.len()
2555 );
2556 self.state.event("prep", note);
2557 }
2558
2559 self.state.event(
2560 "prep",
2561 format!(
2562 "{} candidates, {} judges, base {} ({})",
2563 self.state.candidates.len(),
2564 self.roles.judges.len(),
2565 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2566 self.state.base_branch
2567 ),
2568 );
2569 self.state.prep_complete = Some(true);
2570 self.state.status = RunStatus::Implementing;
2571 self.state.save()?;
2572 Ok(())
2573 }
2574
2575 async fn advise(&mut self) -> Result<()> {
2608 let implement_untouched = self
2609 .state
2610 .candidates
2611 .iter()
2612 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2613 if !self.state.config.graph.advise || self.state.advise_attempted {
2614 return Ok(());
2615 }
2616 if !implement_untouched {
2617 self.state.event(
2618 "advise",
2619 "skipping the design-deliberation stage: at least one \
2620 candidate already shows implementation progress, so this \
2621 run is past the point the stage exists to run before"
2622 .to_owned(),
2623 );
2624 self.state.advise_attempted = true;
2625 self.state.save()?;
2626 return Ok(());
2627 }
2628 let run_id = self.state.id.clone();
2629 let prompts = self.state.config.prompts.clone();
2630 let instruction = self.state.instruction.clone();
2631 let language = self.state.config.graph.language.clone();
2632 let root = self.state.worktree_root();
2633 let n = self.state.config.graph.advisors;
2634 let where_recorded = self.state.dir().join("run.json");
2635
2636 let seats = match self.state.config.advisors() {
2637 Ok(seats) if !seats.is_empty() => seats,
2638 Ok(_) => {
2639 self.state.event(
2640 "advise",
2641 format!(
2642 "[graph] advisors is 0; skipping the design-deliberation \
2643 stage and continuing without a synthesis brief (see {})",
2644 where_recorded.display()
2645 ),
2646 );
2647 self.state.advise_attempted = true;
2648 self.state.save()?;
2649 return Ok(());
2650 }
2651 Err(e) => {
2652 self.state.event(
2653 "advise",
2654 format!(
2655 "could not resolve advisor seats ({e:#}); continuing \
2656 without a design-deliberation brief (see {})",
2657 where_recorded.display()
2658 ),
2659 );
2660 self.state.advise_attempted = true;
2661 self.state.save()?;
2662 return Ok(());
2663 }
2664 };
2665
2666 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2667 let artifacts = agent::artifacts_dir(&self.state.dir());
2668 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2669
2670 let mut jobs = Vec::new();
2671 for (i, spec) in seats.iter().cloned().enumerate() {
2672 let seat_key = format!("advisor-{}", i + 1);
2673 let seat = self.seat(&seat_key, &spec.id);
2674 jobs.push(SeatJob {
2675 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2676 spec,
2677 seat,
2678 cwd: worktrees[i % worktrees.len()].clone(),
2679 timeout,
2680 allow_write: false,
2681 sessions: false,
2682 artifacts: artifacts.clone(),
2683 stem: seat_key,
2684 handover: None,
2685 });
2686 }
2687
2688 self.state.event(
2689 "advise",
2690 format!(
2691 "{} advisor seat(s) sketching a design in parallel",
2692 jobs.len()
2693 ),
2694 );
2695 let mut quota_losses = Vec::new();
2696 let cache = self.state.config.cache_dir();
2697 let ctx = WaveCtx {
2698 carry_seats: false,
2699 pauses: Some((&self.pause, &self.interrupt)),
2700 run: &run_id,
2701 node: "advise",
2702 prompts: &prompts,
2703 cache: cache.as_deref(),
2704 round: None,
2705 };
2706 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2707 let results = ask_json_wave::<Proposal>(
2708 jobs,
2709 Arc::clone(&self.sem),
2710 self.state.config.graph.retries,
2711 &advisor_roster,
2712 &ctx,
2713 &mut quota_losses,
2714 &mut self.state,
2715 &|p: &Proposal| p.validate(),
2716 )
2717 .await;
2718 if self.state.resource_block.is_some() {
2719 return self.stop_for_resource();
2720 }
2721 self.state.quota.extend(quota_losses);
2722
2723 let mut records = Vec::with_capacity(results.len());
2724 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2725 let agent_id = seat.agent.clone();
2726 self.state.seats.insert(seat.key.clone(), seat);
2727 match res {
2728 Ok((proposal, out)) => {
2729 self.state
2730 .event("advise", format!("advisor-{} proposed a design", i + 1));
2731 records.push(advise::AdvisorRecord::proposed(
2732 i + 1,
2733 agent_id,
2734 proposal,
2735 out.duration_ms,
2736 ));
2737 }
2738 Err(e) => {
2739 self.state.event(
2740 "advise",
2741 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2742 );
2743 records.push(advise::AdvisorRecord::failed(
2744 i + 1,
2745 agent_id,
2746 e.to_string(),
2747 ));
2748 }
2749 }
2750 }
2751
2752 let mut advice = advise::Advice {
2753 records,
2754 synthesis: None,
2755 };
2756 if advice.proposals().is_empty() {
2757 self.state.event(
2758 "advise",
2759 "no advisor produced a usable proposal; continuing without a \
2760 synthesis brief"
2761 .to_owned(),
2762 );
2763 } else {
2764 match self
2765 .synthesize_brief(
2766 &advice,
2767 &instruction,
2768 &language,
2769 &worktrees[0],
2770 &artifacts,
2771 &run_id,
2772 &prompts,
2773 cache.as_deref(),
2774 )
2775 .await
2776 {
2777 Ok(Some(text)) => {
2778 self.state.event(
2779 "advise",
2780 "synthesized a design brief for the implementer".to_owned(),
2781 );
2782 advice.synthesis = Some(text);
2783 }
2784 Ok(None) => {
2785 self.state.event(
2786 "advise",
2787 "the synthesis seat produced nothing usable; continuing \
2788 without a design brief"
2789 .to_owned(),
2790 );
2791 }
2792 Err(e) => {
2793 if e.is::<crate::repo_lock::ResourceBlocked>() {
2794 return Err(e);
2795 }
2796 self.state.event(
2797 "advise",
2798 format!("could not synthesize a design brief: {e:#}"),
2799 );
2800 }
2801 }
2802 }
2803 advise::apply_reflection(&mut advice);
2804
2805 self.state.advice = Some(advice);
2806 self.state.advise_attempted = true;
2807 self.state.save()?;
2808 Ok(())
2809 }
2810
2811 #[allow(clippy::too_many_arguments)]
2823 async fn synthesize_brief(
2824 &mut self,
2825 advice: &advise::Advice,
2826 instruction: &str,
2827 language: &str,
2828 cwd: &Path,
2829 artifacts: &Path,
2830 run_id: &str,
2831 prompts: &Prompts,
2832 cache: Option<&Path>,
2833 ) -> Result<Option<String>> {
2834 let chain = agent::pick_chain(
2835 &self.state.config.agents,
2836 self.state.config.roles.synthesizer.as_ref(),
2837 &agent::installed,
2838 "synthesizer",
2839 )?;
2840 let proposals = advice.proposals();
2841 let mut prompt = prompt::with_overlay(
2842 prompt::synthesize_brief(instruction, &proposals, language),
2843 prompts.overlay("advise"),
2844 );
2845 if cache.is_some() {
2846 prompt.push('\n');
2851 prompt.push_str(&prompt::build_cache_note("advise", false));
2852 }
2853 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2854 let mut last = None;
2859 for (n, spec) in chain.iter().enumerate() {
2860 if n > 0 {
2861 self.state
2862 .event("advise", format!("synthesis falling back to {}", spec.id));
2863 }
2864 let mut seat = self.seat("advise-synthesis", &spec.id);
2865 let wait = self.repository_wait();
2866 let outcome = wait
2867 .scope(agent::invoke(
2868 spec,
2869 &mut seat,
2870 &Invocation {
2871 coordination: self.state.config.coordination.as_ref(),
2872 fetched_base: None,
2873 cwd,
2874 prompt: &prompt,
2875 timeout,
2876 allow_write: false,
2877 sessions: false,
2878 artifacts,
2879 stem: &if n == 0 {
2880 "advise-synthesis".to_owned()
2881 } else {
2882 format!("advise-synthesis-{}", spec.id)
2883 },
2884 run: run_id,
2885 node: "advise",
2886 cache_dir: None,
2887 attachments: &[],
2888 writable: &[],
2889 },
2890 ))
2891 .await;
2892 if outcome.is_ok() {
2893 self.state.seats.insert(seat.key.clone(), seat);
2894 }
2895 let advance = agent::chain_advances(&outcome);
2896 last = Some(outcome);
2897 if !advance {
2898 break;
2899 }
2900 }
2901 let out = last.expect("a chain holds at least one agent")?;
2904 if !out.usable() {
2905 return Ok(None);
2906 }
2907 let text =
2908 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2909 Ok((!text.trim().is_empty()).then_some(text))
2910 }
2911
2912 async fn implement(&mut self) -> Result<()> {
2915 self.persist_candidates().await?;
2916 let run_id = self.state.id.clone();
2921 let prompts = self.state.config.prompts.clone();
2922 let todo: Vec<usize> = self
2923 .state
2924 .candidates
2925 .iter()
2926 .enumerate()
2927 .filter(|(_, c)| {
2928 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2929 && c.failed.is_none()
2930 && !c.empty
2931 })
2932 .map(|(i, _)| i)
2933 .collect();
2934 if todo.is_empty() {
2935 return self.after_implement();
2936 }
2937 self.state.status = RunStatus::Implementing;
2938
2939 let language = self.state.config.graph.language.clone();
2940 let github_language = self.state.config.graph.github_language.clone();
2941 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2942 let sessions = self.state.config.graph.sessions;
2943 let artifacts = agent::artifacts_dir(&self.state.dir());
2944 let brief = self
2948 .state
2949 .advice
2950 .as_ref()
2951 .and_then(|a| a.synthesis.as_deref())
2952 .map(str::to_owned);
2953 let attachments = self.state.attachments.clone();
2954
2955 let mut jobs = Vec::new();
2956 for &i in &todo {
2957 let (index, label, worktree) = {
2958 let c = &self.state.candidates[i];
2959 (c.index, c.label, c.worktree.clone())
2960 };
2961 let spec = self.roles.implementers[index].clone();
2962 let seat_key = format!("impl-{label}");
2963 let seat = self.seat(&seat_key, &spec.id);
2964 let instruction = seeded_instruction(&self.state);
2965 jobs.push(SeatJob {
2966 spec,
2967 seat,
2968 prompt: prompt::implement_in(
2969 &instruction,
2970 &worktree.to_string_lossy(),
2971 &language,
2972 &github_language,
2973 brief.as_deref(),
2974 &attachments,
2975 ),
2976 cwd: worktree,
2977 timeout,
2978 allow_write: true,
2979 sessions,
2980 artifacts: artifacts.clone(),
2981 stem: format!("impl-{label}"),
2982 handover: None,
2983 });
2984 }
2985
2986 self.state.event(
2987 "implement",
2988 format!("{} candidates in parallel", jobs.len()),
2989 );
2990 let mut sent = jobs.clone();
2996 let cache = self.state.config.cache_dir();
2997 let ctx = WaveCtx {
2998 carry_seats: false,
2999 pauses: Some((&self.pause, &self.interrupt)),
3000 run: &run_id,
3001 node: "implement",
3002 prompts: &prompts,
3003 cache: cache.as_deref(),
3004 round: None,
3005 };
3006 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3007 if !wave_blocked(&results) {
3008 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3009 .await;
3010 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3011 .await?;
3012 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3013 .await;
3014 }
3015 if !results.is_empty()
3018 && results
3019 .iter()
3020 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3021 {
3022 let losses: Vec<_> = results
3023 .iter()
3024 .map(|(_, seat, out)| {
3025 let AgentOutcome::Quota(o) = out else {
3026 unreachable!()
3027 };
3028 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3029 })
3030 .collect();
3031 self.state.quota.extend(losses.clone());
3032 for (_, seat, _) in results {
3033 self.state.seats.insert(seat.key.clone(), seat);
3034 }
3035 for &i in &todo {
3036 let label = self.state.candidates[i].label;
3037 self.state.interrupted_implementations.insert(label);
3038 self.state
3039 .candidate_preservation
3040 .insert(label, "machine outage awaiting persistence".into());
3041 }
3042 block_machine_outage(&mut self.state, "implement", &losses);
3045 self.state.save()?;
3046 self.persist_candidates().await?;
3047 for &i in &todo {
3048 self.state
3049 .interrupted_implementations
3050 .insert(self.state.candidates[i].label);
3051 }
3052 return self.stop_for_resource();
3053 }
3054 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3055
3056 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3057 && results
3058 .iter()
3059 .any(|(_, _, out)| FailClass::of(out).is_some());
3060 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3061 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3062 if self
3066 .state
3067 .candidate_preservation
3068 .contains_key(&self.state.candidates[i].label)
3069 {
3070 self.state.seats.insert(seat.key.clone(), seat);
3071 }
3072 continue;
3073 }
3074 let seat_key = seat.key.clone();
3075 let agent = seat.agent.clone();
3084 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3085 self.state.seats.insert(seat.key.clone(), seat);
3086 let label = self.state.candidates[i].label;
3087 self.state.interrupted_implementations.remove(&label);
3088
3089 let (summary, duration, failed, verified_claim) = match out {
3090 AgentOutcome::Ok(o) => {
3091 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3092 let failed = (!o.usable()).then(|| {
3093 if o.timed_out {
3094 "agent timed out".to_owned()
3095 } else {
3096 format!(
3097 "agent exited with {}",
3098 crate::notices::exit_status(o.exit_code)
3099 )
3100 }
3101 });
3102 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3103 (text, o.duration_ms, failed, verified_claim)
3104 }
3105 AgentOutcome::Dropped(o) => {
3111 let why = o
3112 .dropped
3113 .as_ref()
3114 .map(|d| d.why.as_str())
3115 .unwrap_or("the CLI ended the stream without delivering its answer");
3116 (
3117 String::new(),
3118 o.duration_ms,
3119 Some(format!("the CLI dropped the stream ({why})")),
3120 None,
3121 )
3122 }
3123 AgentOutcome::Quota(o) => {
3124 self.state.quota.push(QuotaLoss::from_report(
3125 seat_key,
3126 "implement",
3127 o.quota.as_ref(),
3128 ));
3129 (
3130 String::new(),
3131 o.duration_ms,
3132 Some("rate limited (quota); produced no change".to_owned()),
3133 None,
3134 )
3135 }
3136 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3137 (String::new(), 0, Some(e), None)
3138 }
3139 };
3140
3141 let c = &mut self.state.candidates[i];
3144 if !exhausted_the_fallback_chain {
3145 c.agent = agent;
3146 }
3147 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3148 c.duration_ms = duration;
3149 c.failed = failed;
3150 c.verified_noop = verified_claim;
3151 self.state
3152 .candidate_preservation
3153 .insert(label, "awaiting persistence".to_owned());
3154 }
3155 self.state.save()?;
3156 self.persist_candidates().await?;
3157
3158 if resource_blocked {
3159 return self.stop_for_resource();
3160 }
3161 if selected_failed {
3162 self.state.status = RunStatus::Blocked;
3163 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3164 self.state.save()?;
3165 bail!(
3166 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3167 );
3168 }
3169 self.after_implement()
3170 }
3171
3172 async fn persist_candidates(&mut self) -> Result<()> {
3174 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3175 for label in labels {
3176 let i = self
3177 .state
3178 .candidates
3179 .iter()
3180 .position(|c| c.label == label)
3181 .context("pending candidate is missing")?;
3182 let c = self.state.candidates[i].clone();
3183 let interrupted = self.state.interrupted_implementations.contains(&label);
3184 let result = async {
3185 let wait = crate::repo_lock::WaitPolicy::new(
3186 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3187 || false,
3188 );
3189 let _coordination =
3190 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3191 if let Err(e) = git::snapshot_uncommitted(
3194 &c.worktree,
3195 &self
3196 .state
3197 .dir()
3198 .join("artifacts")
3199 .join(format!("candidate-{label}-recovery")),
3200 )
3201 .await
3202 {
3203 self.state.event(
3204 "preserve",
3205 format!("candidate {label} snapshot incomplete: {e:#}"),
3206 );
3207 }
3208 let rescued = git::rescue_commit(
3209 &c.worktree,
3210 &format!("magi: candidate {label} (uncommitted work)"),
3211 )
3212 .await?;
3213 self.state.note_withheld("implement", &rescued.withheld);
3214 let base = &self.state.base_commit;
3215 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3216 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3217 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3218 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3219 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3220 if c.empty && commits > 0 && !patch.trim().is_empty()
3221 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3222 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3223 || self.state.tally.is_some())
3224 {
3225 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3229 }
3230 let c = &mut self.state.candidates[i];
3231 c.commits = commits;
3232 c.stat = stat;
3233 c.files = files;
3234 let no_changes = commits == 0 || patch.trim().is_empty();
3235 c.empty = no_changes && !interrupted;
3236 if interrupted && no_changes {
3237 c.failed = None;
3238 c.verified_noop = None;
3239 }
3240 if !c.empty {
3241 c.failed = None;
3242 c.verified_noop = None;
3243 }
3244 self.state.event(
3245 "preserve",
3246 format!("candidate {label}: {files} files, {commits} commits"),
3247 );
3248 Ok::<_, anyhow::Error>(())
3249 }
3250 .await;
3251 match result {
3252 Ok(()) => {
3253 self.state.candidate_preservation.remove(&label);
3254 if !interrupted || self.state.candidates[i].files > 0 {
3255 self.state.interrupted_implementations.remove(&label);
3256 }
3257 }
3258 Err(e) => {
3259 let reason = format!("candidate {label} preservation failed: {e:#}");
3260 self.state.event("preserve", &reason);
3261 self.state.candidate_preservation.insert(label, reason);
3262 }
3263 }
3264 if self.state.candidate_preservation.is_empty()
3265 && self.state.status == RunStatus::Blocked
3266 && self.state.candidates.iter().all(|c| c.failed.is_none())
3267 {
3268 self.state.status = RunStatus::Implementing;
3272 self.state.event(
3273 "preserve",
3274 "candidate persistence recovered; implementation can continue",
3275 );
3276 }
3277 self.state.save()?;
3278 }
3279 if !self.state.candidate_preservation.is_empty() {
3280 self.state.status = RunStatus::Blocked;
3281 self.state.save()?;
3282 bail!(
3283 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3284 );
3285 }
3286 Ok(())
3287 }
3288
3289 async fn resume_undelivered(
3317 &mut self,
3318 results: &mut [(usize, SeatState, AgentOutcome)],
3319 sent: &[SeatJob],
3320 prompts: &Prompts,
3321 run_id: &str,
3322 ) {
3323 for (wi, seat, out) in results.iter_mut() {
3324 let Some(dropped) = (match &*out {
3325 AgentOutcome::Dropped(o) => o.dropped.clone(),
3326 _ => None,
3327 }) else {
3328 continue;
3329 };
3330 let Some(job) = sent.get(*wi) else { continue };
3331 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3333 self.state.event(
3334 "implement",
3335 format!(
3336 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3337 work is in the tree",
3338 seat.key, dropped.output_tokens, dropped.why
3339 ),
3340 );
3341 continue;
3342 }
3343 if !has_context(&job.spec, seat, job.sessions) {
3351 self.state.event(
3352 "implement",
3353 format!(
3354 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3355 is no session left to resume",
3356 seat.key, dropped.output_tokens, dropped.why
3357 ),
3358 );
3359 continue;
3360 }
3361 self.state.event(
3362 "implement",
3363 format!(
3364 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3365 conversation",
3366 seat.key, dropped.output_tokens, dropped.why
3367 ),
3368 );
3369 let mut retry = job.clone();
3370 retry.seat = seat.clone();
3371 retry.prompt = prompt::resume_after_drop(&dropped.why);
3372 retry.timeout = retry_budget(job.timeout, true);
3373 retry.stem = format!("{}-resume", job.stem);
3374 let cache = self.state.config.cache_dir();
3375 let ctx = WaveCtx {
3376 carry_seats: false,
3377 pauses: Some((&self.pause, &self.interrupt)),
3378 run: run_id,
3379 node: "implement",
3380 prompts,
3381 cache: cache.as_deref(),
3382 round: None,
3383 };
3384 let (resumed_seat, resumed) =
3385 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3386 *seat = resumed_seat;
3387 *out = resumed;
3388 }
3389 }
3390
3391 async fn resume_seat_handovers(
3456 &mut self,
3457 results: &mut [(usize, SeatState, AgentOutcome)],
3458 sent: &mut [SeatJob],
3459 prompts: &Prompts,
3460 run_id: &str,
3461 ) -> Result<()> {
3462 let instruction = seeded_instruction(&self.state);
3463 let language = self.state.config.graph.language.clone();
3464 let github_language = self.state.config.graph.github_language.clone();
3465 let brief = self
3466 .state
3467 .advice
3468 .as_ref()
3469 .and_then(|a| a.synthesis.as_deref())
3470 .map(str::to_owned);
3471 let attachments = self.state.attachments.clone();
3472 for (wi, seat, out) in results.iter_mut() {
3473 let others: BTreeSet<String> = sent
3476 .iter()
3477 .enumerate()
3478 .filter(|(j, _)| j != wi)
3479 .map(|(_, j)| j.spec.id.clone())
3480 .collect();
3481 let Some(job) = sent.get_mut(*wi) else {
3482 continue;
3483 };
3484 let start = self
3489 .roles
3490 .implementer_roster
3491 .iter()
3492 .position(|s| s.id == job.spec.id)
3493 .unwrap_or(0);
3494 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3495 let mut fallback_attempt = 0usize;
3496 let mut prev: Option<FailClass> = None;
3497 while let Some(cur) = FailClass::of(&*out) {
3498 if !should_hand_over(prev.as_ref(), &cur) {
3499 break;
3500 }
3501 let Some(next) =
3502 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3503 .cloned()
3504 else {
3505 break;
3506 };
3507 tried.insert(next.id.clone());
3508 fallback_attempt += 1;
3509
3510 let coordination = self.coordinate(&job.cwd).await?;
3511 if let Ok(r) = git::rescue_commit(
3512 &job.cwd,
3513 &format!(
3514 "magi: candidate {} (uncommitted work before {} fallback)",
3515 seat.key,
3516 if cur == FailClass::Quota {
3517 "quota"
3518 } else {
3519 "handover"
3520 }
3521 ),
3522 )
3523 .await
3524 {
3525 self.state.note_withheld("implement", &r.withheld);
3526 }
3527 drop(coordination);
3528
3529 record_handover(
3530 &mut self.state,
3531 "implement",
3532 &seat.key,
3533 &seat.agent,
3534 &next.id,
3535 &cur,
3536 &fail_reason(&*out),
3537 );
3538 prev = Some(cur.clone());
3539
3540 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3541 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3542 job.spec = next.clone();
3550 let mut retry = job.clone();
3551 retry.seat = new_seat;
3552 retry.prompt = prompt::implement_in(
3553 &instruction,
3554 &job.cwd.to_string_lossy(),
3555 &language,
3556 &github_language,
3557 brief.as_deref(),
3558 &attachments,
3559 );
3560 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3561 let cache = self.state.config.cache_dir();
3562 let ctx = WaveCtx {
3563 carry_seats: false,
3564 pauses: Some((&self.pause, &self.interrupt)),
3565 run: run_id,
3566 node: "implement",
3567 prompts,
3568 cache: cache.as_deref(),
3569 round: None,
3570 };
3571 let (fallback_seat, fallback_out) = run_one(
3572 retry,
3573 Arc::clone(&self.sem),
3574 &ctx,
3575 &mut self.state,
3576 fallback_attempt,
3577 )
3578 .await;
3579 *seat = fallback_seat;
3580 *out = fallback_out;
3581 }
3582 }
3583 Ok(())
3584 }
3585
3586 async fn resume_unconfirmed_commands(
3610 &mut self,
3611 results: &mut [(usize, SeatState, AgentOutcome)],
3612 sent: &[SeatJob],
3613 prompts: &Prompts,
3614 run_id: &str,
3615 ) {
3616 for (wi, seat, out) in results.iter_mut() {
3617 let AgentOutcome::Ok(o) = &*out else {
3618 continue;
3619 };
3620 if !has_unconfirmed_command(&o.commands) {
3621 continue;
3622 }
3623 let Some(job) = sent.get(*wi) else { continue };
3624 if !has_context(&job.spec, seat, job.sessions) {
3625 self.state.event(
3626 "implement",
3627 format!(
3628 "{}: the reply named a command whose own CLI never confirmed the exit \
3629 status of, but there is no session left to resume",
3630 seat.key
3631 ),
3632 );
3633 continue;
3634 }
3635 self.state.event(
3636 "implement",
3637 format!(
3638 "{}: the reply named a command whose own CLI never confirmed the exit \
3639 status of; resuming the conversation",
3640 seat.key
3641 ),
3642 );
3643 let mut retry = job.clone();
3644 retry.seat = seat.clone();
3645 retry.prompt = prompt::resume_incomplete(
3646 "a command in your last reply had no confirmed exit status",
3647 );
3648 retry.timeout = retry_budget(job.timeout, true);
3649 retry.stem = format!("{}-confirm", job.stem);
3650 let cache = self.state.config.cache_dir();
3651 let ctx = WaveCtx {
3652 carry_seats: false,
3653 pauses: Some((&self.pause, &self.interrupt)),
3654 run: run_id,
3655 node: "implement",
3656 prompts,
3657 cache: cache.as_deref(),
3658 round: None,
3659 };
3660 let (resumed_seat, resumed) =
3661 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3662 *seat = resumed_seat;
3663 *out = resumed;
3664 }
3665 }
3666
3667 async fn continue_fix_report(
3688 &mut self,
3689 mut seat: SeatState,
3690 parse_err: String,
3691 job: &SeatJob,
3692 prompts: &Prompts,
3693 run_id: &str,
3694 round: usize,
3695 ) -> (
3696 SeatState,
3697 Option<FixReport>,
3698 Option<String>,
3699 ContinuationRecord,
3700 ) {
3701 let mut last_err = parse_err;
3702 let mut cumulative_wait_ms = 0u64;
3703 let mut attempts = 0usize;
3704 loop {
3705 if !has_context(&job.spec, &seat, job.sessions) {
3706 self.state.event(
3707 "fix",
3708 format!(
3709 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3710 session left to resume into"
3711 ),
3712 );
3713 let outcome = if attempts == 0 {
3714 ContinuationOutcome::NoSession
3715 } else {
3716 ContinuationOutcome::Exhausted
3717 };
3718 return (
3719 seat,
3720 None,
3721 Some(format!("unparsable fix report: {last_err}")),
3722 ContinuationRecord {
3723 attempts,
3724 cumulative_wait_ms,
3725 outcome,
3726 },
3727 );
3728 }
3729 if attempts >= MAX_FIX_CONTINUATIONS {
3730 self.state.event(
3731 "fix",
3732 format!(
3733 "round {round}: fixer's reply still had no adoption report after \
3734 {attempts} continuation(s) ({last_err}); giving up"
3735 ),
3736 );
3737 return (
3738 seat,
3739 None,
3740 Some(format!(
3741 "unparsable fix report after {attempts} continuation(s): {last_err}"
3742 )),
3743 ContinuationRecord {
3744 attempts,
3745 cumulative_wait_ms,
3746 outcome: ContinuationOutcome::Exhausted,
3747 },
3748 );
3749 }
3750 attempts += 1;
3751 self.state.event(
3752 "fix",
3753 format!(
3754 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3755 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3756 ),
3757 );
3758 let mut retry = job.clone();
3759 retry.seat = seat.clone();
3760 retry.prompt = prompt::resume_incomplete(&last_err);
3761 retry.timeout = retry_budget(job.timeout, true);
3762 retry.stem = format!("{}-continue{attempts}", job.stem);
3763 let cache = self.state.config.cache_dir();
3764 let ctx = WaveCtx {
3765 carry_seats: false,
3766 pauses: Some((&self.pause, &self.interrupt)),
3767 run: run_id,
3768 node: "fix",
3769 prompts,
3770 cache: cache.as_deref(),
3771 round: Some(round),
3772 };
3773 let (resumed_seat, resumed_out) = run_one(
3774 retry,
3775 Arc::clone(&self.sem),
3776 &ctx,
3777 &mut self.state,
3778 attempts,
3779 )
3780 .await;
3781 seat = resumed_seat;
3782 match resumed_out {
3783 AgentOutcome::Ok(o) => {
3784 cumulative_wait_ms += o.duration_ms;
3785 match verdict::extract_json::<FixReport>(&o.text) {
3786 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3787 self.state.event(
3788 "fix",
3789 format!(
3790 "round {round}: fixer's adoption report recovered after \
3791 {attempts} continuation(s)"
3792 ),
3793 );
3794 return (
3795 seat,
3796 Some(report),
3797 None,
3798 ContinuationRecord {
3799 attempts,
3800 cumulative_wait_ms,
3801 outcome: ContinuationOutcome::Resumed,
3802 },
3803 );
3804 }
3805 Ok(_) => {
3813 last_err = "the reply parsed, but it reported a command whose own CLI \
3814 never confirmed an exit status"
3815 .to_owned();
3816 }
3817 Err(e) => last_err = e.to_string(),
3818 }
3819 }
3820 AgentOutcome::Quota(o) => {
3821 cumulative_wait_ms += o.duration_ms;
3822 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3823 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3824 self.state.quota.push(loss);
3825 self.state.event(
3826 "fix",
3827 format!(
3828 "round {round}: continuation rate limited (quota); not retrying now"
3829 ),
3830 );
3831 return (
3832 seat,
3833 None,
3834 Some("rate limited (quota) while recovering the fix report".to_owned()),
3835 ContinuationRecord {
3836 attempts,
3837 cumulative_wait_ms,
3838 outcome: ContinuationOutcome::QuotaLost,
3839 },
3840 );
3841 }
3842 AgentOutcome::Dropped(o) => {
3843 cumulative_wait_ms += o.duration_ms;
3844 let why = o
3845 .dropped
3846 .as_ref()
3847 .map(|d| d.why.as_str())
3848 .unwrap_or("the CLI ended the stream without delivering its answer");
3849 last_err = format!("the CLI dropped the stream ({why})");
3850 }
3851 AgentOutcome::Failed(e) => last_err = e,
3852 AgentOutcome::ResourceBlocked(e) => {
3855 return (
3856 seat,
3857 None,
3858 Some(format!("the continuation was not started: {e}")),
3859 ContinuationRecord {
3860 attempts: attempts.saturating_sub(1),
3861 cumulative_wait_ms,
3862 outcome: ContinuationOutcome::NoSession,
3863 },
3864 );
3865 }
3866 }
3867 }
3868 }
3869
3870 fn after_implement(&mut self) -> Result<()> {
3871 if !self.state.candidate_preservation.is_empty() {
3872 bail!("candidate preservation incomplete; judging is unsafe");
3873 }
3874 if self.state.leaks.is_empty() {
3876 let cfg = self.state.config.blind.clone();
3877 let mut leaks = Vec::new();
3878 for c in &self.state.candidates {
3879 let Some(patch) =
3880 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3881 else {
3882 continue;
3883 };
3884 leaks.extend(blind::scan(
3885 &format!("candidate {} patch", c.label),
3886 &patch,
3887 &cfg.vendor_tokens,
3888 ));
3889 }
3890 if !leaks.is_empty() {
3891 let summary = leaks
3892 .iter()
3893 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3894 .collect::<Vec<_>>()
3895 .join(", ");
3896 match cfg.on_leak {
3897 LeakPolicy::Fail => {
3898 self.state.status = RunStatus::Failed;
3899 self.state
3900 .event("blind", format!("vendor text in a patch: {summary}"));
3901 self.state.leaks = leaks;
3902 self.state.save()?;
3903 self.settle_questions();
3904 bail!(
3905 "blind.on_leak = \"fail\" and vendor text reached a \
3906 judged patch: {summary}"
3907 );
3908 }
3909 LeakPolicy::Redact => self.state.event(
3910 "blind",
3911 format!("redacting vendor text for judging: {summary}"),
3912 ),
3913 LeakPolicy::Warn => self.state.event(
3914 "blind",
3915 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3916 ),
3917 }
3918 self.state.leaks = leaks;
3919 }
3920 }
3921
3922 if self.state.viable().is_empty() {
3923 if self.state.all_candidates_verified_noop() {
3924 self.state.status = RunStatus::VerifiedNoop;
3935 self.state.save()?;
3936 self.settle_questions();
3937 return Ok(());
3938 }
3939 self.state.status = RunStatus::Failed;
3940 self.state.save()?;
3941 self.settle_questions();
3942 bail!("no candidate produced a change; nothing to judge");
3943 }
3944 self.state.status = RunStatus::Judging;
3945 self.state.save()?;
3946 Ok(())
3947 }
3948
3949 async fn judge(&mut self) -> Result<()> {
3952 let run_id = self.state.id.clone();
3957 let prompts = self.state.config.prompts.clone();
3958 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3959 return Ok(());
3960 }
3961 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3962 if viable.len() == 1 {
3963 self.state.judge_skipped = true;
3970 self.state.event(
3971 "judge",
3972 format!(
3973 "only candidate {} produced a change; judging skipped",
3974 viable[0].label
3975 ),
3976 );
3977 self.state.save()?;
3978 return Ok(());
3979 }
3980 self.state.status = RunStatus::Judging;
3981
3982 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983 let language = self.state.config.graph.language.clone();
3984 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985 let sessions = self.state.config.graph.sessions;
3986 let artifacts = agent::artifacts_dir(&self.state.dir());
3987 let root = self.state.worktree_root();
3988 let base_short = short(&self.state.base_commit);
3989
3990 let mut jobs = Vec::new();
3991 let mut orders = Vec::new();
3992 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3993 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3994 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3995 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3996 let seat_key = format!("judge-{}", j + 1);
3997 let seat = self.seat(&seat_key, &spec.id);
3998 jobs.push(SeatJob {
3999 prompt: prompt::judge(
4000 &self.state.instruction,
4001 &views,
4002 self.roles.judges.len(),
4003 &base_short,
4004 &language,
4005 ),
4006 spec,
4007 seat,
4008 cwd: root.join(format!("judge-{}", j + 1)),
4009 timeout,
4010 allow_write: false,
4011 sessions,
4012 artifacts: artifacts.clone(),
4013 stem: format!("judge-{}", j + 1),
4014 handover: None,
4015 });
4016 }
4017
4018 self.state.event(
4019 "judge",
4020 format!(
4021 "{} judges ranking {} candidates blind",
4022 jobs.len(),
4023 viable.len()
4024 ),
4025 );
4026 let labels_for_check = labels.clone();
4027 let mut quota_losses = Vec::new();
4028 let cache = self.state.config.cache_dir();
4029 let ctx = WaveCtx {
4030 carry_seats: false,
4031 pauses: Some((&self.pause, &self.interrupt)),
4032 run: &run_id,
4033 node: "judge",
4034 prompts: &prompts,
4035 cache: cache.as_deref(),
4036 round: None,
4037 };
4038 let results = ask_json_wave::<Ranking>(
4039 jobs,
4040 Arc::clone(&self.sem),
4041 self.state.config.graph.retries,
4042 &self.roles.judge_roster,
4043 &ctx,
4044 &mut quota_losses,
4045 &mut self.state,
4046 &move |r: &Ranking| r.validate(&labels_for_check),
4047 )
4048 .await;
4049 if self.state.resource_block.is_some() {
4050 return self.stop_for_resource();
4051 }
4052 self.state.quota.extend(quota_losses);
4053
4054 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4055 let agent_id = seat.agent.clone();
4056 self.state.seats.insert(seat.key.clone(), seat);
4057 let mut record = Judgement {
4058 judge: j + 1,
4059 seat: format!("judge-{}", j + 1),
4060 agent: agent_id,
4061 ranking: Vec::new(),
4062 reasons: BTreeMap::new(),
4063 confidence: None,
4064 strengths: BTreeMap::new(),
4065 order: orders[j].clone(),
4066 failed: None,
4067 duration_ms: 0,
4068 };
4069 match res {
4070 Ok((ranking, out)) => {
4071 record.ranking = ranking.normalized();
4072 record.reasons = ranking.reasons;
4073 record.strengths = ranking.strengths;
4074 record.confidence = ranking.confidence;
4075 record.duration_ms = out.duration_ms;
4076 self.state.event(
4077 "judge",
4078 format!(
4079 "judge {} ranked {}",
4080 j + 1,
4081 record.ranking.iter().collect::<String>()
4082 ),
4083 );
4084 }
4085 Err(e) => {
4086 record.failed = Some(e.to_string());
4087 self.state
4088 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4089 }
4090 }
4091 self.state.judgements.push(record);
4092 self.state.save()?;
4093 }
4094 Ok(())
4095 }
4096
4097 async fn deliberate(&mut self) -> Result<()> {
4100 if self.hold_base_sync().await? {
4101 return Ok(());
4102 }
4103 let run_id = self.state.id.clone();
4108 let prompts = self.state.config.prompts.clone();
4109 let rounds = self.state.config.graph.deliberate_rounds;
4110 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4112 return Ok(());
4113 }
4114 let tops: Vec<char> = self
4115 .state
4116 .judgements
4117 .iter()
4118 .filter_map(|j| j.ranking.first().copied())
4119 .collect();
4120 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4121 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4122 self.state.event(
4123 "deliberate",
4124 format!("judges agreed on {} outright; no deliberation", tops[0]),
4125 );
4126 }
4127 self.state.status = RunStatus::Voting;
4128 self.state.save()?;
4129 return Ok(());
4130 }
4131
4132 self.state.status = RunStatus::Deliberating;
4133 self.state.event(
4134 "deliberate",
4135 format!(
4136 "split: first choices were {} — opening {rounds} round(s)",
4137 tops.iter().collect::<String>()
4138 ),
4139 );
4140
4141 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4142 let language = self.state.config.graph.language.clone();
4143 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4144 let sessions = self.state.config.graph.sessions;
4145 let artifacts = agent::artifacts_dir(&self.state.dir());
4146 let root = self.state.worktree_root();
4147 let base_short = short(&self.state.base_commit);
4148
4149 for round in self.state.deliberation.len() + 1..=rounds {
4153 let mut turns: Vec<DeliberationTurn> = Vec::new();
4154 let mut round_losses = Vec::new();
4155 let mut asked = 0;
4156 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4157 if self.state.judgements[j].failed.is_some() {
4158 continue;
4159 }
4160 let seat_key = format!("judge-{}", j + 1);
4161 let spec = self.occupant(&seat_key, spec);
4162 let mut seat = self.seat(&seat_key, &spec.id);
4163 let transcript = self.transcript(&turns, j);
4164 let build = |context: Option<&str>| {
4165 prompt::deliberate(
4166 &self.state.instruction,
4167 context,
4168 &transcript,
4169 round,
4170 rounds,
4171 &language,
4172 )
4173 };
4174 let block = self.candidate_block(&viable, &base_short);
4175 let full = build(Some(&block));
4176 let text = if has_context(&spec, &seat, sessions) {
4177 build(None)
4178 } else {
4179 full.clone()
4180 };
4181 let job = SeatJob {
4182 spec,
4183 seat: seat.clone(),
4184 prompt: text,
4185 cwd: root.join(format!("judge-{}", j + 1)),
4186 timeout,
4187 allow_write: false,
4188 sessions,
4189 artifacts: artifacts.clone(),
4190 stem: format!("delib-{round}-judge-{}", j + 1),
4191 handover: Some(full),
4192 };
4193 let cache = self.state.config.cache_dir();
4194 let ctx = WaveCtx {
4195 carry_seats: false,
4196 pauses: Some((&self.pause, &self.interrupt)),
4197 run: &run_id,
4198 node: "deliberate",
4199 prompts: &prompts,
4200 cache: cache.as_deref(),
4201 round: None,
4202 };
4203 let mut losses = Vec::new();
4207 asked += 1;
4208 let mut results = ask_wave_with::<String>(
4209 vec![job],
4210 Arc::clone(&self.sem),
4211 0,
4212 &self.roles.judge_roster,
4213 &ctx,
4214 &mut losses,
4215 &mut self.state,
4216 &|text: &str| {
4217 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4218 },
4219 )
4220 .await;
4221 if self.state.resource_block.is_some() {
4222 return self.stop_for_resource();
4223 }
4224 round_losses.extend(losses.iter().cloned());
4225 self.state.quota.extend(losses);
4226 let (updated, res, _) = results.pop().expect("one job in, one result out");
4227 seat = updated;
4228 let agent_id = seat.agent.clone();
4229 self.state.seats.insert(seat.key.clone(), seat);
4230 let body = match res {
4231 Ok((body, _)) => body,
4232 Err(e) => {
4235 self.state
4236 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4237 continue;
4238 }
4239 };
4240 let tentative = verdict::extract_json::<Position>(&body)
4241 .ok()
4242 .and_then(|p| p.tentative)
4243 .and_then(|s| s.trim().chars().next())
4244 .map(|c| c.to_ascii_uppercase());
4245 self.state.event(
4246 "deliberate",
4247 format!(
4248 "round {round}: judge {} now favours {}",
4249 j + 1,
4250 tentative.map_or("—".to_owned(), |c| c.to_string())
4251 ),
4252 );
4253 turns.push(DeliberationTurn {
4254 judge: j + 1,
4255 agent: agent_id,
4256 body: blind::sanitize_prose(&body, &self.state.config.blind),
4257 tentative,
4258 });
4259 }
4260 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4261 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4262 return self.stop_for_resource();
4263 }
4264 self.state
4265 .deliberation
4266 .push(DeliberationRound { round, turns });
4267 self.state.save()?;
4268 }
4269
4270 self.state.status = RunStatus::Voting;
4271 self.state.save()?;
4272 Ok(())
4273 }
4274
4275 async fn vote(&mut self) -> Result<()> {
4278 if self.hold_base_sync().await? {
4279 return Ok(());
4280 }
4281 let run_id = self.state.id.clone();
4286 let prompts = self.state.config.prompts.clone();
4287 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4288 return Ok(());
4289 }
4290 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4291 if viable.len() == 1 {
4292 return Ok(());
4293 }
4294 self.state.status = RunStatus::Voting;
4295
4296 let language = self.state.config.graph.language.clone();
4297 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4298 let sessions = self.state.config.graph.sessions;
4299 let artifacts = agent::artifacts_dir(&self.state.dir());
4300 let root = self.state.worktree_root();
4301 let base_short = short(&self.state.base_commit);
4302 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4303
4304 let mut jobs = Vec::new();
4305 let mut seats_at = Vec::new();
4306 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4307 if self
4308 .state
4309 .judgements
4310 .get(j)
4311 .is_some_and(|r| r.failed.is_some())
4312 {
4313 continue;
4314 }
4315 let seat_key = format!("judge-{}", j + 1);
4316 let spec = self.occupant(&seat_key, spec);
4317 let seat = self.seat(&seat_key, &spec.id);
4318 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4319 let text = if has_context(&spec, &seat, sessions) {
4320 prompt::final_vote(&viable, &language)
4321 } else {
4322 full.clone()
4323 };
4324 jobs.push(SeatJob {
4325 spec,
4326 seat,
4327 prompt: text,
4328 cwd: root.join(format!("judge-{}", j + 1)),
4329 timeout,
4330 allow_write: false,
4331 sessions,
4332 artifacts: artifacts.clone(),
4333 stem: format!("vote-judge-{}", j + 1),
4334 handover: Some(full),
4335 });
4336 seats_at.push(j);
4337 }
4338
4339 self.state.event(
4340 "vote",
4341 format!(
4342 "collecting {} final votes one by one, privately",
4343 jobs.len()
4344 ),
4345 );
4346 let allowed = viable.clone();
4347 let mut quota_losses = Vec::new();
4348 let cache = self.state.config.cache_dir();
4349 let ctx = WaveCtx {
4350 carry_seats: false,
4351 pauses: Some((&self.pause, &self.interrupt)),
4352 run: &run_id,
4353 node: "vote",
4354 prompts: &prompts,
4355 cache: cache.as_deref(),
4356 round: None,
4357 };
4358 let results = ask_json_wave::<FinalVote>(
4359 jobs,
4360 Arc::clone(&self.sem),
4361 self.state.config.graph.retries,
4362 &self.roles.judge_roster,
4363 &ctx,
4364 &mut quota_losses,
4365 &mut self.state,
4366 &move |v: &FinalVote| match v.label() {
4367 Some(c) if allowed.contains(&c) => Ok(()),
4368 other => bail!("vote {other:?} is not one of {allowed:?}"),
4369 },
4370 )
4371 .await;
4372 if self.state.resource_block.is_some() {
4373 return self.stop_for_resource();
4374 }
4375 self.state.quota.extend(quota_losses);
4376
4377 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4378 let agent_id = seat.agent.clone();
4379 self.state.seats.insert(seat.key.clone(), seat);
4380 let initial = self
4381 .state
4382 .judgements
4383 .get(j)
4384 .and_then(|r| r.ranking.first().copied());
4385 let mut record = VoteRecord {
4386 judge: j + 1,
4387 agent: agent_id,
4388 vote: None,
4389 reason: String::new(),
4390 changed: false,
4391 };
4392 match res {
4393 Ok((v, _)) => {
4394 record.vote = v.label();
4395 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4396 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4397 self.state.event(
4398 "vote",
4399 format!(
4400 "judge {} voted {}{}",
4401 j + 1,
4402 record.vote.unwrap_or('?'),
4403 if record.changed { " (changed)" } else { "" }
4404 ),
4405 );
4406 }
4407 Err(e) => {
4408 self.state
4409 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4410 }
4411 }
4412 self.state.votes.push(record);
4413 self.state.save()?;
4414 }
4415 Ok(())
4416 }
4417
4418 fn tally(&mut self) -> Result<()> {
4421 if self.state.tally.is_some() {
4422 return Ok(());
4423 }
4424 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4425 let tops: Vec<char> = self
4426 .state
4427 .judgements
4428 .iter()
4429 .filter_map(|j| j.ranking.first().copied())
4430 .collect();
4431 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4432
4433 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4436 let mut cast: Vec<char> = Vec::new();
4437 for (i, j) in self.state.judgements.iter().enumerate() {
4438 let vote = self
4439 .state
4440 .votes
4441 .iter()
4442 .find(|v| v.judge == i + 1)
4443 .and_then(|v| v.vote)
4444 .or_else(|| j.ranking.first().copied());
4445 if let Some(v) = vote {
4446 *first_choice.entry(v).or_insert(0) += 1;
4447 cast.push(v);
4448 }
4449 }
4450
4451 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4452 for j in &self.state.judgements {
4453 let n = j.ranking.len();
4454 for (pos, label) in j.ranking.iter().enumerate() {
4455 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4456 }
4457 }
4458
4459 let best = first_choice.values().copied().max().unwrap_or(0);
4460 let mut leaders: Vec<char> = first_choice
4461 .iter()
4462 .filter(|(_, v)| **v == best)
4463 .map(|(k, _)| *k)
4464 .collect();
4465 let mut tie_break = None;
4466 if leaders.len() > 1 {
4467 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4468 let borda_leaders: Vec<char> = leaders
4469 .iter()
4470 .copied()
4471 .filter(|l| borda[l] == top_borda)
4472 .collect();
4473 tie_break = Some(if borda_leaders.len() == 1 {
4474 format!(
4475 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4476 leaders.len()
4477 )
4478 } else {
4479 format!(
4480 "{} way tie on both first-choice votes and Borda points, broken by label order",
4481 leaders.len()
4482 )
4483 });
4484 leaders = borda_leaders;
4485 leaders.sort_unstable();
4486 }
4487 let winner = *leaders
4488 .first()
4489 .or(viable.first())
4490 .context("no candidate to declare a winner from")?;
4491
4492 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4493 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4494 let deliberated = !self.state.deliberation.is_empty();
4495
4496 let quota_seats: std::collections::BTreeSet<&str> =
4500 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4501 let mut present = 0usize;
4502 for (i, j) in self.state.judgements.iter().enumerate() {
4503 if quota_seats.contains(j.seat.as_str()) {
4504 continue;
4505 }
4506 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4507 let voted = self
4508 .state
4509 .votes
4510 .iter()
4511 .any(|v| v.judge == i + 1 && v.vote.is_some());
4512 if ranked || voted {
4513 present += 1;
4514 }
4515 }
4516 let needs_quorum = viable.len() > 1;
4522 let judges_total = if needs_quorum {
4523 self.roles.judges.len()
4524 } else {
4525 0
4526 };
4527 let quorum = if needs_quorum {
4528 judges_total / 2 + 1
4529 } else {
4530 0
4531 };
4532 let met_quorum = !needs_quorum || present >= quorum;
4533 let uncontested = (!needs_quorum).then(|| {
4534 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4535 });
4536
4537 self.state.event(
4538 "tally",
4539 match &uncontested {
4540 Some(reason) => format!("winner {winner} — {reason}"),
4541 None => format!(
4542 "winner {winner} — votes {} | initial {} | {} changed | \
4543 {present}/{judges_total} judges{}",
4544 first_choice
4545 .iter()
4546 .map(|(k, v)| format!("{k}:{v}"))
4547 .collect::<Vec<_>>()
4548 .join(" "),
4549 if unanimous_initial {
4550 "unanimous"
4551 } else {
4552 "split"
4553 },
4554 changed_votes,
4555 if met_quorum {
4556 String::new()
4557 } else {
4558 format!(" — below quorum ({quorum} required)")
4559 },
4560 ),
4561 },
4562 );
4563 if !met_quorum {
4564 self.state.event(
4565 "stall",
4566 format!(
4567 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4568 the run stops here, resumable"
4569 ),
4570 );
4571 }
4572 self.state.tally = Some(Tally {
4573 first_choice,
4574 borda,
4575 winner,
4576 rankings: tops.len(),
4577 unanimous_initial,
4578 deliberated,
4579 changed_votes,
4580 unanimous_final,
4581 tie_break,
4582 judges: judges_total,
4583 present,
4584 quorum,
4585 met_quorum,
4586 uncontested,
4587 });
4588 self.state.status = if met_quorum {
4589 RunStatus::Reviewing
4590 } else {
4591 RunStatus::Stalled
4592 };
4593 self.state.save()?;
4594 Ok(())
4595 }
4596
4597 #[allow(clippy::too_many_lines)]
4618 async fn recover_stall(&mut self) -> Result<bool> {
4619 let run_id = self.state.id.clone();
4624 let prompts = self.state.config.prompts.clone();
4625 let quota_seats: BTreeSet<&str> =
4630 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4631 let absent: Vec<String> = self
4632 .state
4633 .judgements
4634 .iter()
4635 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4636 .map(|j| j.seat.clone())
4637 .collect();
4638 if absent.is_empty() {
4639 return Ok(false);
4640 }
4641 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4642 if viable.len() <= 1 {
4643 return Ok(false);
4644 }
4645 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4646 let language = self.state.config.graph.language.clone();
4647 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4648 let sessions = self.state.config.graph.sessions;
4649 let artifacts = agent::artifacts_dir(&self.state.dir());
4650 let root = self.state.worktree_root();
4651 let base_short = short(&self.state.base_commit);
4652 let candidates: Vec<Candidate> = viable.clone();
4653
4654 let mut positions: Vec<usize> = absent
4656 .iter()
4657 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4658 .collect();
4659 if positions.is_empty() {
4660 return Ok(false);
4661 }
4662 positions.sort_unstable();
4663 positions.dedup();
4664
4665 let mut judge_jobs = Vec::new();
4667 for &j in &positions {
4668 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4669 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4670 let seat_key = format!("judge-{}", j + 1);
4671 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4672 let seat = self.seat(&seat_key, &spec.id);
4673 judge_jobs.push(SeatJob {
4674 spec,
4675 seat,
4676 prompt: prompt::judge(
4677 &self.state.instruction,
4678 &views,
4679 self.roles.judges.len(),
4680 &base_short,
4681 &language,
4682 ),
4683 cwd: root.join(seat_key),
4684 timeout,
4685 allow_write: false,
4686 sessions,
4687 artifacts: artifacts.clone(),
4688 stem: format!("judge-{}-recover", j + 1),
4689 handover: None,
4690 });
4691 }
4692
4693 let labels_for_check = labels.clone();
4694 let mut judge_losses = Vec::new();
4695 let retries = self.state.config.graph.retries;
4696 let cache = self.state.config.cache_dir();
4697 let ctx = WaveCtx {
4698 carry_seats: false,
4699 pauses: Some((&self.pause, &self.interrupt)),
4700 run: &run_id,
4701 node: "judge",
4702 prompts: &prompts,
4703 cache: cache.as_deref(),
4704 round: None,
4705 };
4706 let results = ask_json_wave::<Ranking>(
4707 judge_jobs,
4708 Arc::clone(&self.sem),
4709 retries,
4710 &self.roles.judge_roster,
4711 &ctx,
4712 &mut judge_losses,
4713 &mut self.state,
4714 &move |r: &Ranking| r.validate(&labels_for_check),
4715 )
4716 .await;
4717 if self.state.resource_block.is_some() {
4718 self.stop_for_resource()?;
4719 return Ok(false);
4720 }
4721
4722 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4724 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4725 let agent_id = seat.agent.clone();
4726 self.state.seats.insert(seat.key.clone(), seat);
4727 let record = &mut self.state.judgements[j];
4728 match res {
4729 Ok((ranking, out)) => {
4730 record.agent = agent_id;
4731 record.ranking = ranking.normalized();
4732 record.reasons = ranking.reasons;
4733 record.strengths = ranking.strengths;
4734 record.confidence = ranking.confidence;
4735 record.failed = None;
4736 record.duration_ms = out.duration_ms;
4737 recovered.insert(j);
4738 self.state.event(
4739 "recover",
4740 format!("judge {} ranked again after the limit", j + 1),
4741 );
4742 }
4743 Err(e) => {
4744 self.state
4745 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4746 }
4747 }
4748 }
4749
4750 let mut vote_jobs = Vec::new();
4752 let mut vote_pos: Vec<usize> = Vec::new();
4753 for &j in &recovered {
4754 let seat_key = format!("judge-{}", j + 1);
4755 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4756 let seat = self.seat(&seat_key, &spec.id);
4757 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4758 let text = if has_context(&spec, &seat, sessions) {
4759 prompt::final_vote(&labels, &language)
4760 } else {
4761 full.clone()
4762 };
4763 vote_jobs.push(SeatJob {
4764 spec,
4765 seat,
4766 prompt: text,
4767 cwd: root.join(seat_key),
4768 timeout,
4769 allow_write: false,
4770 sessions,
4771 artifacts: artifacts.clone(),
4772 stem: format!("vote-judge-{}-recover", j + 1),
4773 handover: Some(full),
4774 });
4775 vote_pos.push(j);
4776 }
4777 let allowed = labels.clone();
4778 let mut vote_losses = Vec::new();
4779 let vote_retries = self.state.config.graph.retries;
4780 let vote_cache = self.state.config.cache_dir();
4781 let ctx = WaveCtx {
4782 carry_seats: false,
4783 pauses: Some((&self.pause, &self.interrupt)),
4784 run: &run_id,
4785 node: "vote",
4786 prompts: &prompts,
4787 cache: vote_cache.as_deref(),
4788 round: None,
4789 };
4790 let votes = ask_json_wave::<FinalVote>(
4791 vote_jobs,
4792 Arc::clone(&self.sem),
4793 vote_retries,
4794 &self.roles.judge_roster,
4795 &ctx,
4796 &mut vote_losses,
4797 &mut self.state,
4798 &move |v: &FinalVote| match v.label() {
4799 Some(c) if allowed.contains(&c) => Ok(()),
4800 other => bail!("vote {other:?} is not one of {allowed:?}"),
4801 },
4802 )
4803 .await;
4804 if self.state.resource_block.is_some() {
4805 self.stop_for_resource()?;
4806 return Ok(false);
4807 }
4808 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4809 let agent_id = seat.agent.clone();
4810 self.state.seats.insert(seat.key.clone(), seat);
4811 match res {
4812 Ok((v, _)) => {
4813 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4814 rec.vote = v.label();
4815 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4816 } else {
4817 self.state.votes.push(VoteRecord {
4818 judge: j + 1,
4819 agent: agent_id,
4820 vote: v.label(),
4821 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4822 changed: false,
4823 });
4824 }
4825 self.state.event(
4826 "recover",
4827 format!("judge {} voted again after the limit", j + 1),
4828 );
4829 }
4830 Err(e) => {
4831 self.state
4832 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4833 }
4834 }
4835 }
4836
4837 let recovered_keys: BTreeSet<String> = recovered
4841 .iter()
4842 .map(|&j| format!("judge-{}", j + 1))
4843 .collect();
4844 self.state
4845 .quota
4846 .retain(|q| !recovered_keys.contains(&q.seat));
4847 for loss in judge_losses.into_iter().chain(vote_losses) {
4851 if recovered_keys.contains(&loss.seat) {
4852 continue;
4853 }
4854 self.state.quota.retain(|q| q.seat != loss.seat);
4855 self.state.quota.push(loss);
4856 }
4857
4858 self.state.tally = None;
4860 self.tally()?;
4861 Ok(self
4862 .state
4863 .tally
4864 .as_ref()
4865 .map(|t| t.met_quorum)
4866 .unwrap_or(false))
4867 }
4868
4869 async fn graft(&mut self) -> Result<()> {
4885 let Err(e) = self.graft_inner().await else {
4886 return Ok(());
4887 };
4888 if e.is::<crate::repo_lock::ResourceBlocked>() {
4889 return Err(e);
4890 }
4891 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4892 self.state.status = RunStatus::Blocked;
4893 self.state.event("graft", format!("blocked: {e:#}"));
4894 self.state.save()?;
4895 return Err(e);
4896 }
4897 if let Some(rec) = self.state.graft.clone()
4902 && !rec.outcome.settled()
4903 {
4904 if let Some(winner) = self.state.winner().cloned() {
4905 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4906 if re.is::<crate::repo_lock::ResourceBlocked>() {
4907 return Err(re);
4908 }
4909 self.state.status = RunStatus::Blocked;
4910 self.state.event("graft", format!("blocked: {re:#}"));
4911 self.state.save()?;
4912 return Err(re);
4913 }
4914 }
4915 if let Some(g) = self.state.graft.as_mut() {
4916 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4917 }
4918 }
4919 self.state.event("graft", format!("skipped: {e:#}"));
4920 if let Err(e) = self.state.save() {
4921 tracing::warn!("save after graft: {e:#}");
4922 }
4923 Ok(())
4924 }
4925
4926 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4929 let _coordination = self.coordinate(wt).await?;
4930 crate::recovery::restore_repair(wt, pre)
4931 .await
4932 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4933 }
4934
4935 fn graft_skipped(&mut self, why: &str) {
4937 let message = format!("skipped: {why}");
4938 if !self
4939 .state
4940 .events
4941 .iter()
4942 .any(|e| e.node == "graft" && e.message == message)
4943 {
4944 self.state.event("graft", message);
4945 }
4946 }
4947
4948 async fn graft_inner(&mut self) -> Result<()> {
4949 if let Some(rec) = self.state.graft.clone()
4952 && !rec.outcome.settled()
4953 {
4954 if self.state.reviews.is_empty()
4958 && let Some(winner) = self.state.winner().cloned()
4959 {
4960 self.graft_restore(&winner.worktree, &rec.pre).await?;
4961 }
4962 if rec.outcome == GraftOutcome::WaitingForRestore {
4963 self.state.graft = None;
4964 self.state.save()?;
4965 } else {
4966 if let Some(g) = self.state.graft.as_mut() {
4967 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4968 }
4969 self.state.event(
4970 "graft",
4971 "rolled back: the previous round was interrupted; the winner is untouched",
4972 );
4973 self.state.save()?;
4974 return Ok(());
4975 }
4976 }
4977 let plan = match graft::plan(&self.state) {
4978 Ok(plan) => plan,
4979 Err(skip) => {
4980 if skip != graft::Skip::AlreadyDone {
4981 self.graft_skipped(skip.reason());
4982 self.state.save()?;
4983 }
4984 return Ok(());
4985 }
4986 };
4987 let Some(winner) = self.state.winner().cloned() else {
4988 return Ok(());
4989 };
4990 let wt = winner.worktree.clone();
4991 if !git::is_clean(&wt).await? {
4992 self.graft_skipped("the winner's worktree has uncommitted changes");
4993 self.state.save()?;
4994 return Ok(());
4995 }
4996 let pre = git::rev_parse(&wt, "HEAD").await?;
4997 self.state.graft = Some(GraftRecord {
5000 pre: pre.clone(),
5001 post: None,
5002 lines: None,
5003 limit: None,
5004 agent: None,
5005 outcome: GraftOutcome::Running,
5006 requested: graft::requested(&plan),
5007 adoption: None,
5008 adoption_error: None,
5009 });
5010 self.state.event(
5011 "graft",
5012 format!(
5013 "porting strengths of candidate(s) {} onto {}",
5014 plan.strengths
5015 .iter()
5016 .map(|(l, _, _)| l.to_string())
5017 .collect::<Vec<_>>()
5018 .join(", "),
5019 plan.winner
5020 ),
5021 );
5022 self.state.save()?;
5023
5024 let fix_prompt = prompt::graft(
5025 &self.state.instruction,
5026 plan.winner,
5027 &plan.strengths,
5028 &self.state.config.graph.language,
5029 );
5030 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5031 let sessions = self.state.config.graph.sessions;
5032 let artifacts = agent::artifacts_dir(&self.state.dir());
5033 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5034 let (_, seat, out) = self
5035 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5036 SeatJob {
5037 prompt: fix_prompt.clone(),
5038 spec,
5039 seat,
5040 cwd: wt.clone(),
5041 timeout,
5042 allow_write: true,
5043 sessions,
5044 artifacts: artifacts.clone(),
5045 stem: "graft".to_owned(),
5046 handover: None,
5047 }
5048 })
5049 .await;
5050 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5051 self.state.graft.as_mut().expect("graft record").outcome =
5054 GraftOutcome::WaitingForRestore;
5055 return self.stop_for_resource();
5056 }
5057 let mut why: Option<String> = None;
5058 match &out {
5059 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5060 why = Some("the fixer produced nothing usable".to_owned());
5061 }
5062 AgentOutcome::Ok(_) => {}
5063 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5064 AgentOutcome::Quota(o) => {
5065 self.state.quota.push(QuotaLoss::from_report(
5066 seat.key.clone(),
5067 "graft",
5068 o.quota.as_ref(),
5069 ));
5070 why = Some("rate limited (quota)".to_owned());
5071 }
5072 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5073 why = Some(format!(
5074 "the fixer failed: {}",
5075 e.lines()
5076 .next()
5077 .unwrap_or("")
5078 .chars()
5079 .take(160)
5080 .collect::<String>()
5081 ));
5082 }
5083 }
5084 let agent_id = seat.agent.clone();
5085 self.state.seats.insert(seat.key.clone(), seat);
5086
5087 let mut post = None;
5088 let mut lines = None;
5089 let _coordination = self.coordinate(&wt).await?;
5090 if why.is_none() {
5091 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5092 if let Ok(r) = &rescued {
5093 self.state.note_withheld("graft", &r.withheld);
5094 }
5095 let head = git::rev_parse(&wt, "HEAD").await?;
5096 let dirty = match &rescued {
5100 Ok(r) => git::status_porcelain(&wt)
5101 .await
5102 .map(|st| {
5103 st.lines().any(|l| {
5104 !r.withheld
5105 .iter()
5106 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5107 })
5108 })
5109 .unwrap_or(true),
5110 Err(_) => true,
5111 };
5112 if dirty {
5113 why = Some("the fixer left changes that could not be committed".to_owned());
5114 } else if head == pre {
5115 why = Some("the fixer changed nothing".to_owned());
5116 } else {
5117 lines = git::changed_lines(&wt, &pre, &head).await;
5118 post = Some(head);
5119 }
5120 }
5121 let (adoption, adoption_error) = match &out {
5122 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5123 _ => (None, None),
5124 };
5125 let requested = graft::requested(&plan);
5126 let outcome = match why {
5127 None => {
5128 self.state.event(
5129 "graft",
5130 format!(
5131 "applied: {}",
5132 lines.map_or_else(
5133 || "changed lines unavailable".to_owned(),
5134 |n| format!("{n} changed lines")
5135 )
5136 ),
5137 );
5138 GraftOutcome::Applied
5139 }
5140 Some(why) => {
5141 crate::recovery::restore_repair(&wt, &pre)
5144 .await
5145 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5146 self.state.event(
5147 "graft",
5148 format!("rolled back: {why}; the winner is untouched"),
5149 );
5150 GraftOutcome::RolledBack(why)
5151 }
5152 };
5153 self.state.graft = Some(GraftRecord {
5154 pre,
5155 post,
5156 lines,
5157 limit: None,
5158 agent: Some(agent_id),
5159 outcome,
5160 requested,
5161 adoption,
5162 adoption_error,
5163 });
5164 self.state.save()?;
5165 Ok(())
5166 }
5167
5168 async fn fold_losers(&mut self) -> Result<()> {
5171 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5172 return Ok(());
5173 };
5174 if !self.state.candidate_preservation.is_empty() {
5175 bail!("candidate preservation incomplete; refusing cleanup");
5176 }
5177 let repo = self.state.repo.clone();
5178 let _coordination = self.coordinate(&repo).await?;
5179 let mut folded = Vec::new();
5180 for i in 0..self.state.candidates.len() {
5181 let c = &self.state.candidates[i];
5182 if c.label == winner || c.folded {
5183 continue;
5184 }
5185 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5186 remove_preserved_worktree(&repo, &wt).await?;
5187 if git::branch_exists(&repo, &branch).await?
5188 && !git::branch_delete(&repo, &branch).await?
5189 {
5190 bail!("candidate branch removal refused");
5191 }
5192 self.state.candidates[i].folded = true;
5193 self.state.save()?;
5194 folded.push(label.to_string());
5195 }
5196 let root = self.state.worktree_root();
5198 for j in 1..=self.roles.judges.len() {
5199 let wt = root.join(format!("judge-{j}"));
5200 if wt.exists() {
5201 remove_preserved_worktree(&repo, &wt).await?;
5202 }
5203 }
5204 if self.state.config.graph.advise {
5207 for k in 1..=self.state.config.graph.advisors {
5208 let wt = root.join(format!("advisor-{k}"));
5209 if wt.exists() {
5210 remove_preserved_worktree(&repo, &wt).await?;
5211 }
5212 }
5213 }
5214 if !folded.is_empty() {
5215 self.state
5216 .event("fold", format!("folded candidates {}", folded.join(", ")));
5217 self.state.save()?;
5218 }
5219 Ok(())
5220 }
5221
5222 async fn hold_base_sync(&mut self) -> Result<bool> {
5225 self.hold_base_sync_as(None).await
5226 }
5227
5228 async fn hold_base_sync_kind(
5229 &mut self,
5230 kind: crate::base_sync_recovery::StopKind,
5231 ) -> Result<()> {
5232 self.hold_base_sync_as(Some(kind)).await?;
5233 Ok(())
5234 }
5235
5236 async fn hold_base_sync_as(
5237 &mut self,
5238 kind: Option<crate::base_sync_recovery::StopKind>,
5239 ) -> Result<bool> {
5240 if !self.state.base_sync_blocked() {
5241 self.state.base_sync_recovery.stop = None;
5242 return Ok(false);
5243 }
5244 let head = match self.state.winner() {
5245 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5246 .await
5247 .unwrap_or_default(),
5248 None => String::new(),
5249 };
5250 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5251 && !authorization.consumed
5252 && authorization.stop.head != head
5253 {
5254 authorization.refuse(format!(
5255 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5256 authorization.stop.head
5257 ));
5258 }
5259 self.state.record_base_sync_stop(&head);
5260 if let Some(kind) = kind {
5261 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5262 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5263 stop.kind = kind;
5264 }
5265 }
5266 self.state.save()?;
5267 Ok(true)
5268 }
5269
5270 async fn sync_to_base(&mut self) -> Result<()> {
5312 if self.state.status == RunStatus::AlreadyInBase {
5313 return Ok(());
5314 }
5315 self.hold_base_sync().await?;
5316 let conflicted = self
5317 .state
5318 .base_sync
5319 .as_ref()
5320 .is_some_and(|s| s.conflict.is_some());
5321 let Some(winner) = self.state.winner().cloned() else {
5322 return Ok(());
5323 };
5324
5325 let repo = self.state.repo.clone();
5326 let remote = self.state.config.merge.remote.clone();
5327 let base_branch = self.state.base_branch.clone();
5328 let tracking = format!("{remote}/{base_branch}");
5329
5330 let fetched_base =
5331 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5332 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5337 return Ok(());
5338 };
5339
5340 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5341 self.state.ensure_base_episode();
5342 let mut pending = self
5343 .state
5344 .base_sync_recovery
5345 .operations
5346 .last()
5347 .filter(|o| !o.finished)
5348 .cloned();
5349 let scratch = self.state.dir().join("base-sync");
5350 if git::rebase_in_progress(&scratch).await {
5351 if pending.is_none()
5354 && self.state.base_sync_recovery.operations.is_empty()
5355 && !conflicted
5356 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5357 .await
5358 .ok()
5359 .as_deref()
5360 == Some(head.as_str())
5361 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5362 && git::is_clean(&winner.worktree).await?
5363 {
5364 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5365 if self.state.base_sync.is_none() {
5366 self.state.base_sync = Some(BaseSync {
5367 tip: saved_tip.clone(),
5368 behind: 0,
5369 attempts: 0,
5370 conflict: None,
5371 already_in: None,
5372 });
5373 }
5374 self.state.reserve_base_operation(&head, &saved_tip);
5375 self.state.event(
5376 "base-recovery",
5377 "imported standing legacy rebase as a spent synchronization reservation",
5378 );
5379 self.state.save()?;
5380 pending = self.state.base_sync_recovery.operations.last().cloned();
5381 }
5382 let expected = pending.as_ref().map(|op| op.tip.as_str());
5383 if expected.is_none()
5384 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5385 {
5386 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5387 if self.state.base_sync.is_none() {
5388 self.state.base_sync = Some(BaseSync {
5389 tip: tip.clone(),
5390 behind: 0,
5391 attempts: 0,
5392 conflict: None,
5393 already_in: None,
5394 });
5395 }
5396 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5397 self.state.event("land", why);
5398 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5399 .await?;
5400 return Ok(());
5401 }
5402 }
5403 if let Some(op) = &pending {
5404 let mut replay_proven =
5406 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5407 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5408 let wait = repository_wait(&self.state, None);
5409 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5410 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5411 Ok(true) => {
5412 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5413 self.state.save()?;
5414 git::worktree_remove(&repo, &scratch).await.ok();
5415 replay_proven = true;
5416 }
5417 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5418 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5419 }
5420 }
5421 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5422 if self.state.foreign_checkout.is_some() {
5423 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5424 .await
5425 .ok()
5426 .as_deref()
5427 == Some(op.head.as_str())
5428 && git::rev_parse(&repo, &format!("{head}^2"))
5429 .await
5430 .ok()
5431 .as_deref()
5432 == Some(op.tip.as_str());
5433 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5434 .await
5435 .is_ok_and(|m| m.is_empty())
5436 && let Ok(added) =
5437 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5438 {
5439 replay_proven = true;
5440 for commit in added {
5441 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5442 replay_proven = false;
5443 break;
5444 }
5445 }
5446 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5447 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5448 }
5449 }
5450 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5454 || (head != op.head
5455 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5456 .await?
5457 .is_empty())
5458 || (!git::is_clean(&winner.worktree).await?
5459 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5460 .await
5461 .is_ok_and(|o| o.ok()))
5462 || !replay_proven
5463 {
5464 let why = format!(
5465 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5466 op.id
5467 );
5468 let sync = self
5469 .state
5470 .base_sync
5471 .as_mut()
5472 .context("reserved synchronization has no lifetime accounting")?;
5473 sync.conflict = Some(why.clone());
5474 self.state.event("land", why);
5475 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5476 .await?;
5477 return Ok(());
5478 }
5479 if head != op.head {
5480 git::sync_to_head(&winner.worktree).await?;
5481 }
5482 tip = op.tip.clone();
5483 }
5484 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5485 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5486 if pending.is_none()
5487 && !conflicted
5488 && fetched_base
5489 && self
5490 .state
5491 .base_sync_recovery
5492 .episodes
5493 .last()
5494 .is_some_and(|e| e.boundary.is_some())
5495 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5496 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5497 && owner.runs.last() == Some(&self.state.id)
5498 && owner.status != crate::queue::TaskStatus::Done
5499 && !owner.fresh_start
5500 && owner.review_branch.is_none()
5501 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5502 && let Some(approval) = owner.resumption.as_ref()
5503 && approval.instruction == owner.instruction
5504 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5505 && git::is_clean(&winner.worktree).await?
5506 && self.state.renew_base_episode(&task, approval, &head, &tip)
5507 {
5508 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5509 self.state.save()?;
5510 }
5511
5512 if (behind > 0 || conflicted || head == tip)
5521 && self
5522 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5523 .await?
5524 {
5525 return Ok(());
5526 }
5527 if conflicted && self.state.base_recovery_pending() {
5528 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5529 let owner =
5530 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5531 let owner_valid = owner.as_ref().is_ok_and(|t| {
5532 t.runs.last() == Some(&self.state.id)
5533 && t.status != crate::queue::TaskStatus::Done
5534 && !t.fresh_start
5535 && t.review_branch.is_none()
5536 });
5537 let invalidation = if !owner_valid {
5538 Some("latest task ownership changed")
5539 } else if !owner.as_ref().is_ok_and(|t| {
5540 authorization
5541 .generation
5542 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5543 }) {
5544 Some("owner approval generation changed")
5545 } else if let Some(scope) = &authorization.scope {
5546 if scope.tip != tip {
5547 Some("fetched base changed")
5548 } else if !self
5549 .state
5550 .base_sync_recovery
5551 .episodes
5552 .last()
5553 .is_some_and(|e| e.id == scope.episode)
5554 {
5555 Some("active synchronization interval changed")
5556 } else if !owner
5557 .as_ref()
5558 .is_ok_and(|t| t.instruction == scope.instruction)
5559 {
5560 Some("task instruction changed")
5561 } else {
5562 None
5563 }
5564 } else {
5565 None
5566 };
5567 if let Some(reason) = invalidation {
5568 let authorization = self
5569 .state
5570 .base_sync_recovery
5571 .authorizations
5572 .last_mut()
5573 .unwrap();
5574 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5575 self.state.save()?;
5576 return Ok(());
5577 }
5578 if !fetched_base
5579 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5580 || !git::is_clean(&winner.worktree).await?
5581 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5582 {
5583 return Ok(());
5584 }
5585 }
5586 let recovery = conflicted && self.state.consume_base_recovery(&head);
5587 if conflicted {
5588 self.state.save()?;
5590 if !recovery {
5591 return Ok(());
5592 }
5593 }
5594
5595 if behind == 0 {
5596 if let Some(from) = self
5605 .state
5606 .rebase_fixes
5607 .iter()
5608 .rev()
5609 .find_map(|r| r.from.clone())
5610 && from != head
5611 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5612 .await
5613 .is_ok_and(|o| o.ok())
5614 {
5615 git::sync_to_head(&winner.worktree).await?;
5616 }
5617 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5623 if let Some(why) = &conflict {
5624 self.state.status = RunStatus::Blocked;
5625 self.state.event("land", why.clone());
5626 }
5627 self.state.base_sync = Some(BaseSync {
5628 tip,
5629 behind: 0,
5630 attempts,
5631 conflict,
5632 already_in: None,
5633 });
5634 if pending.is_some() {
5635 self.state.finish_base_operation(Some(&head));
5636 }
5637 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5638 .await?;
5639 self.state.save()?;
5640 return Ok(());
5641 }
5642
5643 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5644 let why = format!(
5645 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5646 synchronization(s); synchronizing again would only race it",
5647 winner.branch
5648 );
5649 self.state.status = RunStatus::Blocked;
5650 self.state.base_sync = Some(BaseSync {
5651 tip,
5652 behind,
5653 attempts,
5654 conflict: Some(why.clone()),
5655 already_in: None,
5656 });
5657 self.state.event("land", why);
5658 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5659 .await?;
5660 self.state.save()?;
5661 return Ok(());
5662 }
5663
5664 if self.state.foreign_checkout.is_some() {
5665 let wait = repository_wait(&self.state, None);
5668 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5669 if self.state.base_sync.is_none() {
5670 self.state.base_sync = Some(BaseSync {
5671 tip: tip.clone(),
5672 behind,
5673 attempts: 0,
5674 conflict: None,
5675 already_in: None,
5676 });
5677 }
5678 self.state.reserve_base_operation(&head, &tip);
5679 self.state.save()?;
5680 let merged = git::git_raw(
5681 &winner.worktree,
5682 &[
5683 "-c",
5684 "core.hooksPath=/dev/null",
5685 "merge",
5686 "--no-ff",
5687 "--no-autostash",
5688 "--no-edit",
5689 "-m",
5690 &format!("Merge {tracking} before verifying"),
5691 &tip,
5692 ],
5693 )
5694 .await?;
5695 let conflict = if merged.ok() {
5696 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5697 self.state.event(
5698 "land",
5699 format!("merged {tracking} into {} before verifying", winner.branch),
5700 );
5701 None
5702 } else {
5703 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5704 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5705 }
5706 let why = format!(
5707 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5708 winner.branch,
5709 merged.stderr.chars().take(600).collect::<String>()
5710 );
5711 self.state.status = RunStatus::Blocked;
5712 self.state.event("land", &why);
5713 Some(why)
5714 };
5715 let synced = conflict.is_none();
5716 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5717 self.state
5718 .finish_base_operation(synced.then_some(fresh.as_str()));
5719 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5720 self.state.base_sync = Some(BaseSync {
5721 tip,
5722 behind: if synced { 0 } else { behind },
5723 attempts,
5724 conflict,
5725 already_in: None,
5726 });
5727 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5728 .await?;
5729 self.state.save()?;
5730 return Ok(());
5731 }
5732
5733 self.state.event(
5734 "land",
5735 format!(
5736 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5737 winner.branch
5738 ),
5739 );
5740 self.state.save()?;
5741
5742 let branch_tracking = format!("{remote}/{}", winner.branch);
5748 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5749 let remote_tip = if self.state.task_pr_branch().is_none()
5750 && matches!(&fetched_branch, Ok(o) if o.ok())
5751 {
5752 git::rev_parse(&repo, &branch_tracking).await.ok()
5753 } else {
5754 None
5755 };
5756 if let Some(theirs) = &remote_tip
5760 && !git::is_ancestor(&repo, theirs, &head).await
5761 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5762 .await
5763 .is_ok_and(|missing| missing.is_empty())
5764 {
5765 let why = format!(
5766 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5767 them",
5768 short(theirs),
5769 winner.branch
5770 );
5771 self.state.status = RunStatus::Blocked;
5772 self.state.base_sync = Some(BaseSync {
5773 tip,
5774 behind,
5775 attempts,
5776 conflict: Some(why.clone()),
5777 already_in: None,
5778 });
5779 self.state.event("land", why);
5780 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5781 .await?;
5782 self.state.save()?;
5783 return Ok(());
5784 }
5785
5786 if self.state.base_sync.is_none() {
5787 self.state.base_sync = Some(BaseSync {
5788 tip: tip.clone(),
5789 behind,
5790 attempts: 0,
5791 conflict: None,
5792 already_in: None,
5793 });
5794 }
5795 self.state.reserve_base_operation(&head, &tip);
5796 self.state.save()?;
5797 let scratch = self.state.dir().join("base-sync");
5798 let rebased =
5799 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5800 .await
5801 {
5802 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5803 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5804 Err(e) => Err(e),
5805 };
5806 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5807 let stop_kind = match &rebased {
5808 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5809 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5810 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5811 };
5812 match rebased {
5813 Ok(None) => {
5814 git::sync_to_head(&winner.worktree).await?;
5818 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5819 self.state.finish_base_operation(Some(&fresh));
5820 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5821 let mut conflict = None;
5822 if let Some(pinned) = &remote_tip {
5823 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5824 match pushed {
5825 Ok(o) if o.ok() => self.state.event(
5826 "land",
5827 format!("pushed rebased {} to {remote}", winner.branch),
5828 ),
5829 Ok(o) => {
5830 conflict = Some(format!(
5831 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5832 winner.branch,
5833 short(pinned),
5834 o.stderr.chars().take(600).collect::<String>()
5835 ));
5836 }
5837 Err(e) => {
5838 conflict = Some(format!(
5839 "rebased {} locally but could not push it: {e:#}",
5840 winner.branch
5841 ));
5842 }
5843 }
5844 }
5845 if let Some(why) = &conflict {
5846 self.state.status = RunStatus::Blocked;
5847 self.state.event("land", why.clone());
5848 }
5849 self.state.base_sync = Some(BaseSync {
5850 tip: tip.clone(),
5851 behind: 0,
5852 attempts,
5853 conflict,
5854 already_in: None,
5855 });
5856 self.state
5857 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5858 }
5859 Ok(Some(conflict)) => {
5860 self.state.finish_base_operation(None);
5861 let why = format!(
5862 "{} conflicts with {tracking} and did not rebase: {}",
5863 winner.branch,
5864 conflict.chars().take(600).collect::<String>()
5865 );
5866 self.state.status = RunStatus::Blocked;
5867 self.state.base_sync = Some(BaseSync {
5868 tip,
5869 behind,
5870 attempts,
5871 conflict: Some(why.clone()),
5872 already_in: None,
5873 });
5874 self.state.event("land", why);
5875 }
5876 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883 Err(e) => {
5884 self.state.finish_base_operation(None);
5885 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5886 self.state.status = RunStatus::Blocked;
5887 self.state.base_sync = Some(BaseSync {
5888 tip,
5889 behind,
5890 attempts,
5891 conflict: Some(why.clone()),
5892 already_in: None,
5893 });
5894 self.state.event("land", why);
5895 }
5896 }
5897 if !self.state.base_sync_blocked() {
5898 self.state.base_sync_recovery.stop = None;
5899 }
5900 self.hold_base_sync_kind(stop_kind).await?;
5901 self.state.save()?;
5902 Ok(())
5903 }
5904
5905 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5911 if self.state.task_pr_branch().is_some() {
5914 return None;
5915 }
5916 let repo = self.state.repo.clone();
5917 let remote = self.state.config.merge.remote.clone();
5918 let fetched = git::fetch(&repo, &remote, branch).await;
5919 if !matches!(&fetched, Ok(o) if o.ok()) {
5920 return None;
5921 }
5922 let branch_tracking = format!("{remote}/{branch}");
5923 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5924 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5925 return None;
5926 }
5927 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5928 .await
5929 .is_ok_and(|missing| missing.is_empty())
5930 {
5931 return Some(format!(
5932 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5933 them",
5934 short(&theirs)
5935 ));
5936 }
5937 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5938 Ok(o) if o.ok() => {
5939 self.state
5940 .event("land", format!("pushed rebased {branch} to {remote}"));
5941 None
5942 }
5943 Ok(o) => Some(format!(
5944 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5945 someone may have pushed): {}",
5946 short(&theirs),
5947 o.stderr.chars().take(600).collect::<String>()
5948 )),
5949 Err(e) => Some(format!(
5950 "{branch} is rebased locally but could not be pushed: {e:#}"
5951 )),
5952 }
5953 }
5954
5955 async fn settle_already_in(
5972 &mut self,
5973 branch: &str,
5974 tip: &str,
5975 head: &str,
5976 attempts: usize,
5977 behind: usize,
5978 ) -> Result<bool> {
5979 let repo = self.state.repo.clone();
5980 let remote = self.state.config.merge.remote.clone();
5981 let start = self.state.base_commit.clone();
5982 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5983 Ok(Some(e)) => e,
5984 Ok(None) => return Ok(false),
5985 Err(e) => {
5986 tracing::warn!("already-in-base check for {branch}: {e:#}");
5987 return Ok(false);
5988 }
5989 };
5990 let mut verified = vec![head.to_owned()];
5991 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5992 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5993 if matches!(&fetched, Ok(o) if o.ok())
5994 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5995 && theirs != head
5996 {
5997 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5998 Ok(Some(_)) => verified.push(theirs),
5999 _ => return Ok(false),
6000 }
6001 }
6002 let base_branch = self.state.base_branch.clone();
6003 let message = format!(
6004 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6005 land",
6006 evidence.names(),
6007 evidence.proof.as_str()
6008 );
6009 let closed =
6010 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6011 .await;
6012 match closed {
6013 Ok(Ok(url)) => self
6014 .state
6015 .event("land", format!("closed {url}: superseded on {base_branch}")),
6016 Ok(Err(why)) => self
6017 .state
6018 .event("land", format!("did not close a pull request: {why}")),
6019 Err(e) => {
6020 let why = format!(
6021 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6022 not be closed ({e:#}); resume to retry"
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip: tip.to_owned(),
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6034 .await?;
6035 self.state.save()?;
6036 return Ok(true);
6037 }
6038 }
6039 self.state.status = RunStatus::AlreadyInBase;
6040 self.state.base_sync_recovery.stop = None;
6041 self.state.base_sync = Some(BaseSync {
6042 tip: tip.to_owned(),
6043 behind,
6044 attempts,
6045 conflict: None,
6046 already_in: Some(evidence),
6047 });
6048 self.state.event("land", message);
6049 self.state.save()?;
6050 self.settle_questions();
6051 Ok(true)
6052 }
6053
6054 fn landing_base(&self) -> String {
6064 self.state
6065 .base_sync
6066 .as_ref()
6067 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6068 }
6069
6070 pub async fn fix_selected(
6103 &mut self,
6104 ids: &[String],
6105 reason: &str,
6106 allow_stale: bool,
6107 ) -> Result<()> {
6108 let reason = reason.trim();
6109 if reason.is_empty() {
6110 bail!("a fix request needs a reason — that is the operator's own record of why");
6111 }
6112 if ids.is_empty() {
6113 bail!("no finding id given");
6114 }
6115 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6116 bail!(
6117 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6118 has already concluded — can be given a targeted fix. A run still \
6119 in progress should simply be resumed; a `merged` run's branch has \
6120 already landed, so its answer is a fresh `magi review <branch>`, \
6121 not reopening this run's own record",
6122 self.state.id,
6123 self.state.status.as_str()
6124 );
6125 }
6126 let Some(winner) = self.state.winner().cloned() else {
6127 bail!("run {} has no winning candidate to fix", self.state.id);
6128 };
6129 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6130 bail!(
6131 "branch `{}` no longer exists; this run cannot be extended",
6132 winner.branch
6133 );
6134 }
6135 let home = crate::run::home();
6136 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6137 bail!(
6138 "run {} is currently being worked on by another magi process",
6139 self.state.id
6140 );
6141 }
6142 let _claim = FixClaim::acquire(&self.state.dir())?;
6148
6149 let mut seen = BTreeSet::new();
6153 let mut findings = Vec::new();
6154 let mut missing = Vec::new();
6155 for id in ids {
6156 if !seen.insert(id.clone()) {
6157 continue;
6158 }
6159 match self.state.finding(id) {
6160 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6161 id: f.id.clone(),
6162 severity: f.severity,
6163 reviewer_vote: rec.vote,
6164 round: round.round,
6165 round_head: round.head.clone(),
6166 reviewer: rec.reviewer,
6167 agent: rec.agent.clone(),
6168 file: f.file.clone(),
6169 line: f.line,
6170 title: f.title.clone(),
6171 detail: f.detail.clone(),
6172 outcome: OperatorFixOutcome::Pending,
6173 }),
6174 None => missing.push(id.clone()),
6175 }
6176 }
6177 if !missing.is_empty() {
6178 bail!(
6179 "unknown finding id(s): {}; nothing was changed",
6180 missing.join(", ")
6181 );
6182 }
6183
6184 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6185 let stale_details: Vec<(String, String)> = findings
6186 .iter()
6187 .filter(|f| f.round_head != head_at_request)
6188 .map(|f| (f.id.clone(), f.round_head.clone()))
6189 .collect();
6190 let stale = !stale_details.is_empty();
6191 if stale && !allow_stale {
6192 bail!(
6193 "the branch has moved since some finding(s) were raised — {} — now \
6194 at {}; pass --allow-stale to fix anyway, or re-run review first",
6195 stale_details
6196 .iter()
6197 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6198 .collect::<Vec<_>>()
6199 .join(", "),
6200 short(&head_at_request)
6201 );
6202 }
6203
6204 let request = OperatorFixRequest {
6205 requested_at: Timestamp::now(),
6206 reason: reason.to_owned(),
6207 findings,
6208 head_at_request: head_at_request.clone(),
6209 allow_stale,
6210 stale,
6211 fix: None,
6212 result_head: None,
6213 follow_up_review_run: None,
6214 };
6215 self.state.event(
6216 "fix",
6217 format!(
6218 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6219 request.findings.len(),
6220 request
6221 .findings
6222 .iter()
6223 .map(|f| f.id.as_str())
6224 .collect::<Vec<_>>()
6225 .join(", "),
6226 ),
6227 );
6228 self.state.operator_fixes.push(request);
6235 self.state.save()?;
6236 let request_index = self.state.operator_fixes.len() - 1;
6237
6238 let repo = self.state.repo.clone();
6247 let coordination = self.coordinate(&repo).await?;
6248 if winner.worktree.exists() {
6249 let dirty = git::git(
6252 &winner.worktree,
6253 &["status", "--porcelain", "--untracked-files=all"],
6254 )
6255 .await?;
6256 let only_withheld = dirty.lines().all(|l| {
6257 l.strip_prefix("?? ")
6258 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6259 });
6260 if !only_withheld {
6261 bail!(
6262 "`{}` has uncommitted changes; refusing to touch it — commit or \
6263 discard them first",
6264 winner.worktree.display()
6265 );
6266 }
6267 git::worktree_remove(&self.state.repo, &winner.worktree)
6268 .await
6269 .ok();
6270 }
6271 let fix_worktree = self.state.worktree_root().join("operator-fix");
6272 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6273 git::git(
6274 &self.state.repo,
6275 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6276 )
6277 .await
6278 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6279 if !git::is_clean(&fix_worktree).await? {
6280 git::worktree_remove(&self.state.repo, &fix_worktree)
6281 .await
6282 .ok();
6283 bail!(
6284 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6285 winner.branch
6286 );
6287 }
6288 let repo_for_setup = self.state.repo.clone();
6289 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6290 Ok(()) => {
6291 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6292 }
6293 Err(e) => Err(e),
6294 };
6295 if let Err(e) = setup {
6296 git::worktree_remove(&self.state.repo, &fix_worktree)
6297 .await
6298 .ok();
6299 return Err(e);
6300 }
6301
6302 self.state.resource_block = None;
6305
6306 let run_id = self.state.id.clone();
6307 let prompts = self.state.config.prompts.clone();
6308 let language = self.state.config.graph.language.clone();
6309 let sessions = self.state.config.graph.sessions;
6310 let artifacts = agent::artifacts_dir(&self.state.dir());
6311 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6312 .findings
6313 .iter()
6314 .map(|f| Finding {
6315 id: f.id.clone(),
6316 severity: f.severity,
6317 file: f.file.clone(),
6318 line: f.line,
6319 title: f.title.clone(),
6320 detail: f.detail.clone(),
6321 })
6322 .collect();
6323 let fix_prompt = prompt::operator_fix_in(
6324 &self.state.instruction,
6325 &finding_list,
6326 reason,
6327 &stale_details,
6328 &head_at_request,
6329 &language,
6330 &self.state.config.graph.github_language,
6331 );
6332 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6333 drop(coordination);
6334 let (job, seat, out) = self
6335 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6336 prompt: fix_prompt.clone(),
6337 spec,
6338 seat,
6339 cwd: fix_worktree.clone(),
6340 timeout,
6341 allow_write: true,
6342 sessions,
6343 artifacts: artifacts.clone(),
6344 stem: "operator-fix".to_owned(),
6345 handover: None,
6346 })
6347 .await;
6348 let agent_id = seat.agent.clone();
6349
6350 let mut fix = FixRecord {
6351 agent: agent_id,
6352 addressed: Vec::new(),
6353 rejected: Vec::new(),
6354 notes: String::new(),
6355 committed: false,
6356 failed: None,
6357 duration_ms: 0,
6358 continuation: None,
6359 };
6360 let mut final_seat = seat.clone();
6361 match out {
6362 AgentOutcome::Ok(o) => {
6363 fix.duration_ms = o.duration_ms;
6364 let parsed = verdict::extract_json::<FixReport>(&o.text);
6365 let incomplete_reason = match &parsed {
6366 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6367 "the reply parsed, but it reported a command whose own CLI \
6368 never confirmed an exit status"
6369 .to_owned(),
6370 ),
6371 Ok(_) => None,
6372 Err(e) => Some(e.to_string()),
6373 };
6374 match incomplete_reason {
6375 None => {
6376 let report = parsed.expect("checked Ok above");
6377 fix.addressed = report.addressed;
6378 fix.rejected = report.rejected;
6379 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6380 }
6381 Some(reason) => {
6382 let (resumed_seat, resolved, failure, cont) = self
6383 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6384 .await;
6385 fix.duration_ms += cont.cumulative_wait_ms;
6386 fix.continuation = Some(cont);
6387 final_seat = resumed_seat;
6388 match resolved {
6389 Some(report) => {
6390 fix.addressed = report.addressed;
6391 fix.rejected = report.rejected;
6392 fix.notes =
6393 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6394 }
6395 None => fix.failed = failure,
6396 }
6397 }
6398 }
6399 }
6400 AgentOutcome::Dropped(o) => {
6401 fix.duration_ms = o.duration_ms;
6402 let why = o
6403 .dropped
6404 .as_ref()
6405 .map(|d| d.why.as_str())
6406 .unwrap_or("the CLI ended the stream without delivering its answer");
6407 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6408 }
6409 AgentOutcome::Quota(o) => {
6410 self.state.quota.push(QuotaLoss::from_report(
6411 final_seat.key.clone(),
6412 "fix",
6413 o.quota.as_ref(),
6414 ));
6415 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6416 }
6417 AgentOutcome::Failed(e) => fix.failed = Some(e),
6418 AgentOutcome::ResourceBlocked(e) => {
6419 fix.failed = Some(format!("the fixer was not started: {e}"));
6420 }
6421 }
6422 if fix.continuation.is_none() {
6425 fix.continuation = Some(ContinuationRecord::not_needed());
6426 }
6427 self.state.seats.insert(final_seat.key.clone(), final_seat);
6428
6429 let rescue_message = format!(
6430 "magi: operator-selected fix ({}) (uncommitted work)",
6431 self.state.operator_fixes[request_index]
6432 .findings
6433 .iter()
6434 .map(|f| f.id.as_str())
6435 .collect::<Vec<_>>()
6436 .join(", ")
6437 );
6438 let repo = self.state.repo.clone();
6439 let coordination = self.coordinate(&repo).await?;
6440 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6441 self.state.note_withheld("fix", &r.withheld);
6442 }
6443 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6444 fix.committed = after != head_at_request;
6445 git::worktree_remove(&self.state.repo, &fix_worktree)
6446 .await
6447 .ok();
6448 drop(coordination);
6449
6450 self.state.event(
6451 "fix",
6452 match &fix.failed {
6453 Some(reason) => format!(
6454 "operator fix: adoption report was lost ({reason}); {}",
6455 if fix.committed {
6456 "committed"
6457 } else {
6458 "NO new commit"
6459 }
6460 ),
6461 None => format!(
6462 "operator fix: {} addressed, {} rejected, {}",
6463 fix.addressed.len(),
6464 fix.rejected.len(),
6465 if fix.committed {
6466 "committed"
6467 } else {
6468 "NO new commit"
6469 }
6470 ),
6471 },
6472 );
6473
6474 for f in &mut self.state.operator_fixes[request_index].findings {
6481 f.outcome = if fix.failed.is_some() {
6482 OperatorFixOutcome::Unreported
6483 } else if fix.addressed.contains(&f.id) {
6484 OperatorFixOutcome::Addressed
6485 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6486 OperatorFixOutcome::Rejected { why: r.why.clone() }
6487 } else {
6488 OperatorFixOutcome::Unreported
6489 };
6490 }
6491
6492 let committed = fix.committed;
6493 if committed {
6494 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6495 }
6496 self.state.operator_fixes[request_index].fix = Some(fix);
6497 self.state.save()?;
6500 if self.state.resource_block.is_some() {
6501 return self.stop_for_resource();
6502 }
6503
6504 if committed {
6505 self.state.event(
6506 "fix",
6507 format!(
6508 "operator fix committed {}; opening a follow-up review-only run",
6509 short(&after)
6510 ),
6511 );
6512 let origin =
6515 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6516 match Self::review(
6517 &self.state.repo,
6518 &winner.branch,
6519 self.state.config.clone(),
6520 origin,
6521 )
6522 .await
6523 {
6524 Ok(mut follow_up) => {
6525 follow_up.state.event(
6526 "start",
6527 format!(
6528 "requested by an operator fix on run {} for finding(s) {}",
6529 self.state.id,
6530 self.state.operator_fixes[request_index]
6531 .findings
6532 .iter()
6533 .map(|f| f.id.as_str())
6534 .collect::<Vec<_>>()
6535 .join(", "),
6536 ),
6537 );
6538 follow_up.state.save()?;
6539 let follow_up_id = follow_up.state.id.clone();
6540 let adopted = match crate::direct::adopt(
6543 &follow_up.state,
6544 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6545 ) {
6546 Ok(a) => a,
6547 Err(e) => {
6548 self.state.event(
6551 "fix",
6552 format!(
6553 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6554 ),
6555 );
6556 self.state.save()?;
6557 return Ok(());
6558 }
6559 };
6560 let executed = follow_up.execute().await;
6561 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6562 if let Some(a) = adopted {
6563 a.finish(&follow_up.state, executed);
6564 }
6565 if let Some(e) = failure {
6566 self.state.event(
6567 "fix",
6568 format!(
6569 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6570 ),
6571 );
6572 }
6573 self.state.operator_fixes[request_index].follow_up_review_run =
6574 Some(follow_up_id);
6575 }
6576 Err(e) => {
6577 self.state.event(
6578 "fix",
6579 format!("committed the fix but could not open a follow-up review: {e:#}"),
6580 );
6581 }
6582 }
6583 self.state.save()?;
6584 }
6585
6586 Ok(())
6587 }
6588
6589 async fn ask_fixer(
6609 &mut self,
6610 winner: &Candidate,
6611 node: &'static str,
6612 round: Option<usize>,
6613 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6614 ) -> (SeatJob, SeatState, AgentOutcome) {
6615 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6616 self.ask_fixer_among(attempts, node, round, None, build)
6617 .await
6618 }
6619
6620 async fn ask_fixer_among(
6625 &mut self,
6626 attempts: Vec<(AgentSpec, String)>,
6627 node: &'static str,
6628 round: Option<usize>,
6629 restore: Option<(&Path, &str)>,
6630 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6631 ) -> (SeatJob, SeatState, AgentOutcome) {
6632 let run_id = self.state.id.clone();
6633 let prompts = self.state.config.prompts.clone();
6634 let cache = self.state.config.cache_dir();
6635 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6636 let mut last = None;
6637 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6638 let seat = self.seat(&key, &spec.id);
6639 let mut job = build(spec.clone(), seat);
6640 if i > 0 {
6641 job.stem = format!("{}-{}", job.stem, spec.id);
6642 }
6643 let ctx = WaveCtx {
6644 carry_seats: false,
6645 pauses: Some((&self.pause, &self.interrupt)),
6646 run: &run_id,
6647 node,
6648 prompts: &prompts,
6649 cache: cache.as_deref(),
6650 round,
6651 };
6652 let (seat, out) =
6653 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6654 let advances = match &out {
6657 AgentOutcome::Ok(o) => agent::output_advances(o),
6658 AgentOutcome::ResourceBlocked(_) => false,
6659 _ => true,
6660 };
6661 if let Some(next) = ids.get(i + 1)
6662 && advances
6663 {
6664 self.state.seats.insert(seat.key.clone(), seat.clone());
6665 let class =
6666 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6667 let reason = match &out {
6668 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6669 other => fail_reason(other),
6670 };
6671 if let Some((wt, rev)) = restore
6675 && let Err(e) = self.graft_restore(wt, rev).await
6676 {
6677 let why =
6678 format!("could not restore the worktree before the next fixer: {e:#}");
6679 tracing::warn!("{why}");
6680 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6681 AgentOutcome::ResourceBlocked(why)
6682 } else {
6683 AgentOutcome::Failed(why)
6684 };
6685 last = Some((job, seat, out));
6686 break;
6687 }
6688 record_handover(
6689 &mut self.state,
6690 node,
6691 &seat.key,
6692 &spec.id,
6693 next,
6694 &class,
6695 &reason,
6696 );
6697 continue;
6698 }
6699 last = Some((job, seat, out));
6700 break;
6701 }
6702 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6703 if let AgentOutcome::Quota(output) = &out
6704 && node != "graft"
6705 {
6706 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6711 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6712 self.state.quota.push(loss);
6713 self.state.seats.insert(seat.key.clone(), seat.clone());
6714 self.state.status = RunStatus::Blocked;
6715 return (
6716 job,
6717 seat,
6718 AgentOutcome::ResourceBlocked(
6719 "fixer unavailable; restore agent availability and resume".into(),
6720 ),
6721 );
6722 }
6723 (job, seat, out)
6724 }
6725
6726 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6737 use crate::recovery::{Mode, Repair};
6738 if !self.state.reviews.is_empty() {
6739 return Ok(());
6740 }
6741 let Some(mut rec) = self.state.recovery.clone() else {
6742 return Ok(());
6743 };
6744 if rec.mode != Mode::VerifyRepair {
6745 return Ok(());
6746 }
6747 let wt = winner.worktree.clone();
6748 let mut head = git::rev_parse(&wt, "HEAD").await?;
6749 match rec.repair.clone() {
6750 Repair::Done { .. } => return Ok(()),
6751 Repair::WaitingForRestore { before } => {
6752 self.graft_restore(&wt, &before).await?;
6753 head = before;
6754 rec.repair = Repair::NotStarted;
6755 self.state.recovery = Some(rec.clone());
6756 self.state.save()?;
6757 }
6758 Repair::Started { before } => {
6759 if let Err(e) = self.graft_restore(&wt, &before).await {
6763 if e.is::<crate::repo_lock::ResourceBlocked>() {
6764 return Err(e);
6765 }
6766 return self
6767 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6768 .await;
6769 }
6770 rec.repair = Repair::Done {
6771 before: before.clone(),
6772 after: before,
6773 committed: false,
6774 failed: Some("interrupted; rolled back".to_owned()),
6775 };
6776 self.state
6777 .event("recovery", "the repair was interrupted and rolled back");
6778 self.state.recovery = Some(rec);
6779 self.state.save()?;
6780 return Ok(());
6781 }
6782 Repair::NotStarted => {}
6783 }
6784 if crate::recovery::recheck(&mut rec, &head) {
6785 self.state.event("recovery", rec.summary());
6786 self.state.recovery = Some(rec);
6787 self.state.save()?;
6788 return Ok(());
6789 }
6790 rec.repair = Repair::Started {
6791 before: head.clone(),
6792 };
6793 self.state.recovery = Some(rec.clone());
6794 self.state.save()?;
6795
6796 let language = self.state.config.graph.language.clone();
6797 let sessions = self.state.config.graph.sessions;
6798 let artifacts = agent::artifacts_dir(&self.state.dir());
6799 let fix_prompt = prompt::recovery_fix(
6800 &self.state.instruction,
6801 &rec,
6802 self.state.config.graph.review_rounds,
6803 &language,
6804 );
6805 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6806 self.state.event(
6807 "recovery",
6808 format!(
6809 "repair: the fixer starts from the failed verification on {}",
6810 short(&head)
6811 ),
6812 );
6813 let (_job, seat, out) = self
6814 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6815 prompt: fix_prompt.clone(),
6816 spec,
6817 seat,
6818 cwd: wt.clone(),
6819 timeout,
6820 allow_write: true,
6821 sessions,
6822 artifacts: artifacts.clone(),
6823 stem: "fix-recovery".to_owned(),
6824 handover: None,
6825 })
6826 .await;
6827 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6828 rec.repair = Repair::WaitingForRestore { before: head };
6829 self.state.recovery = Some(rec);
6830 return self.stop_for_resource();
6831 }
6832 let failed = match &out {
6833 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6834 AgentOutcome::Quota(o) => {
6835 self.state.quota.push(QuotaLoss::from_report(
6836 seat.key.clone(),
6837 "fix",
6838 o.quota.as_ref(),
6839 ));
6840 Some("rate limited (quota); the fixer could not run".to_owned())
6841 }
6842 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6843 AgentOutcome::Failed(e) => Some(e.clone()),
6844 };
6845 self.state.seats.insert(seat.key.clone(), seat);
6846 let coordination = self.coordinate(&wt).await?;
6847 let failed = match failed {
6848 Some(why) => {
6850 if let Err(e) = git::restore_to(&wt, &head).await {
6851 return self
6852 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6853 .await;
6854 }
6855 Some(why)
6856 }
6857 None => {
6858 if let Ok(r) = git::rescue_commit(
6859 &wt,
6860 "magi: recovery repair of the failed verification (uncommitted work)",
6861 )
6862 .await
6863 {
6864 self.state.note_withheld("fix", &r.withheld);
6865 }
6866 None
6867 }
6868 };
6869 let after = git::rev_parse(&wt, "HEAD").await?;
6870 drop(coordination);
6871 self.state.event(
6872 "recovery",
6873 match &failed {
6874 Some(why) => format!("repair: nothing kept ({why})"),
6875 None if after != head => {
6876 format!("repair: committed {}", short(&after))
6877 }
6878 None => "repair: the fixer made no commit".to_owned(),
6879 },
6880 );
6881 rec.repair = Repair::Done {
6882 before: head.clone(),
6883 committed: after != head,
6884 after,
6885 failed,
6886 };
6887 self.state.recovery = Some(rec);
6888 self.state.save()?;
6889 Ok(())
6890 }
6891
6892 async fn stop_recovery(
6895 &mut self,
6896 mut rec: crate::recovery::RecoveryRecord,
6897 head: &str,
6898 why: String,
6899 ) -> Result<()> {
6900 rec.repair = crate::recovery::Repair::Started {
6902 before: head.to_owned(),
6903 };
6904 self.state.recovery = Some(rec);
6905 self.state
6906 .event("recovery", format!("repair could not be undone: {why}"));
6907 self.state.status = RunStatus::Blocked;
6908 self.state.save()?;
6909 bail!("recovery repair could not be rolled back: {why}")
6910 }
6911
6912 async fn review_loop(&mut self) -> Result<()> {
6913 if self.hold_base_sync().await? {
6914 return Ok(());
6915 }
6916 loop {
6917 if self
6918 .state
6919 .base_sync
6920 .as_ref()
6921 .is_some_and(|s| s.conflict.is_some())
6922 {
6923 return Ok(());
6924 }
6925 if self.catchup_repairable() {
6926 let record = self.state.reviews.last().expect("repairable round").clone();
6927 let winner = self
6928 .state
6929 .winner()
6930 .cloned()
6931 .context("catch-up repair has no winner")?;
6932 if !self.fix_review_round(&winner, record).await? {
6933 return Ok(());
6934 }
6935 }
6936 self.review_loop_pass().await?;
6937 if !self.catchup_repairable() {
6938 return Ok(());
6939 }
6940 }
6941 }
6942
6943 fn catchup_repairable(&self) -> bool {
6946 self.state.reviews.len() < self.state.config.graph.review_rounds
6947 && self.state.reviews.last().is_some_and(|r| {
6948 !r.clean
6949 && r.blocking == 0
6950 && r.fix.is_none()
6951 && r.e2e_status() == E2eStatus::Failed
6952 })
6953 }
6954
6955 async fn review_loop_pass(&mut self) -> Result<()> {
6956 if self
6961 .state
6962 .base_sync
6963 .as_ref()
6964 .is_some_and(|s| s.conflict.is_some())
6965 {
6966 return Ok(());
6967 }
6968 let run_id = self.state.id.clone();
6973 let prompts = self.state.config.prompts.clone();
6974 let Some(winner) = self.state.winner().cloned() else {
6975 return Ok(());
6976 };
6977 let granted_review = self
6978 .state
6979 .repair_grants
6980 .last()
6981 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6982 .cloned();
6983 let fresh_review = granted_review
6984 .as_ref()
6985 .is_some_and(|g| self.state.reviews.len() == g.review_start);
6986 let max_rounds = granted_review
6987 .as_ref()
6988 .map_or(self.state.config.graph.review_rounds, |g| {
6989 g.review_start.saturating_add(1)
6990 });
6991 if let Some(status) =
7001 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7002 {
7003 if status == RunStatus::Gating {
7006 self.record_contested_handoff();
7007 }
7008 self.state.status = status;
7009 self.state.save()?;
7010 return Ok(());
7011 }
7012 self.state.status = RunStatus::Reviewing;
7013 if !fresh_review
7023 && self.state.reviews.last().is_some_and(|r| {
7024 r.e2e_status() == E2eStatus::ResourceBlocked
7025 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7026 })
7027 {
7028 let shell = self.state.config.shell();
7029 return self
7030 .stop_unresolved_review(
7031 "the last round's own verification never resolved",
7032 &shell,
7033 &winner.worktree,
7034 )
7035 .await;
7036 }
7037
7038 if granted_review.is_none() {
7039 self.recovery_repair(&winner).await?;
7040 }
7041 if self.resource_halt()? {
7042 return Ok(());
7043 }
7044
7045 let repo = self.state.repo.clone();
7046 let root = self.state.worktree_root();
7047 let language = self.state.config.graph.language.clone();
7048 let sessions = self.state.config.graph.sessions;
7049 let artifacts = self.merge_repair_artifacts();
7050 let base = self.landing_base();
7051 let base_short = short(&base);
7052 let reviewers = self.roles.reviewers.clone();
7053 let shell = self.state.config.shell();
7054
7055 for round in (self.state.reviews.len() + 1)..=max_rounds {
7056 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7057 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7058 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7059 let prev_verification = self
7068 .state
7069 .reviews
7070 .last()
7071 .and_then(|r| r.verification_summary(&head));
7072 let prior_section = match (&self.state.recovery, round) {
7075 (Some(rec), 1) if self.state.reviews.is_empty() => {
7076 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7077 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7078 Some(crate::recovery::head_of(
7079 &git::diff(&winner.worktree, reviewed, "HEAD")
7080 .await
7081 .unwrap_or_default(),
7082 crate::recovery::SINCE_DIFF_MAX,
7083 ))
7084 }
7085 _ => None,
7086 };
7087 Some(prompt::recovery_review(rec, since.as_deref()))
7088 }
7089 _ => None,
7090 };
7091
7092 let mut jobs = Vec::new();
7096 let coordination = self.coordinate(&repo).await?;
7097 for (r, spec) in reviewers.iter().cloned().enumerate() {
7098 let wt = root.join(format!("review-{}", r + 1));
7099 if wt.exists() {
7102 git::reset_detached(&wt, &head).await?;
7103 } else {
7104 git::worktree_add_detached(&repo, &wt, &head).await?;
7105 }
7106 self.setup_seat_worktree(&repo, &wt, false).await?;
7107 let seat_key = format!("review-{}", r + 1);
7108 let spec = pick_start_spec(
7111 &self.roles.reviewer_roster,
7112 spec,
7113 self.state.seat_history.get(&seat_key),
7114 );
7115 let seat = self.seat(&seat_key, &spec.id);
7116 jobs.push(SeatJob {
7117 prompt: prompt::review(&prompt::ReviewCtx {
7118 instruction: &self.state.instruction,
7119 branch: &winner.branch,
7120 base_short: &base_short,
7121 stat: &stat,
7122 patch: &patch,
7123 verification: prev_verification.as_ref(),
7124 prior: prior_section.as_deref(),
7125 reviewers: reviewers.len(),
7126 round,
7127 rounds: max_rounds,
7128 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7131 lens: Lens::for_seat(r),
7132 language: &language,
7133 github_language: &self.state.config.graph.github_language,
7134 }),
7135 spec,
7136 seat,
7137 cwd: wt,
7138 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7139 allow_write: false,
7140 sessions,
7141 artifacts: artifacts.clone(),
7142 stem: format!("review-{round}-{}", r + 1),
7143 handover: None,
7144 });
7145 }
7146
7147 drop(coordination);
7148 self.state.event(
7149 "review",
7150 format!(
7151 "round {round}: {} reviewers on {}",
7152 jobs.len(),
7153 short(&head)
7154 ),
7155 );
7156 let mut quota_losses = Vec::new();
7157 let review_retries = self.state.config.graph.retries;
7158 let review_cache = self.state.config.cache_dir();
7159 let ctx = WaveCtx {
7160 carry_seats: true,
7161 pauses: Some((&self.pause, &self.interrupt)),
7162 run: &run_id,
7163 node: "review",
7164 prompts: &prompts,
7165 cache: review_cache.as_deref(),
7166 round: Some(round),
7167 };
7168 let results = ask_json_wave::<Review>(
7169 jobs,
7170 Arc::clone(&self.sem),
7171 review_retries,
7172 &self.roles.reviewer_roster,
7173 &ctx,
7174 &mut quota_losses,
7175 &mut self.state,
7176 &|_: &Review| Ok(()),
7177 )
7178 .await;
7179 if self.state.resource_block.is_some() {
7180 return self.stop_for_resource();
7181 }
7182 let round_quota_missing = quota_losses.len();
7186 self.state.quota.extend(quota_losses);
7187
7188 let mut records = Vec::new();
7189 let mut all_findings = Vec::new();
7190 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7191 let agent_id = seat.agent.clone();
7192 self.state.seats.insert(seat.key.clone(), seat);
7193 let mut record = ReviewRecord {
7194 reviewer: r + 1,
7195 agent: agent_id,
7196 summary: String::new(),
7197 findings: Vec::new(),
7198 vote: None,
7199 failed: None,
7200 duration_ms: 0,
7201 attempts,
7207 };
7208 match res {
7209 Ok((review, out)) => {
7210 record.summary =
7218 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7219 record.vote = Some(review.vote);
7220 record.duration_ms = out.duration_ms;
7221 for (n, mut f) in review.findings.into_iter().enumerate() {
7222 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7225 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7226 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7227 f.file = f
7233 .file
7234 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7235 all_findings.push(f.clone());
7236 record.findings.push(f);
7237 }
7238 self.state.event(
7239 "review",
7240 format!(
7241 "round {round}: reviewer {} voted {} with {} finding(s)",
7242 r + 1,
7243 review.vote.label(),
7244 record.findings.len()
7245 ),
7246 );
7247 }
7248 Err(e) => {
7249 record.failed = Some(e.to_string());
7250 self.state.event(
7251 "review",
7252 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7253 );
7254 }
7255 }
7256 records.push(record);
7257 }
7258
7259 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7266 let vote_split =
7267 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7268 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7269 if vote_split {
7270 self.state.event(
7271 "review",
7272 format!(
7273 "round {round}: votes split ({}) — one round of reconsideration",
7274 initial_votes
7275 .iter()
7276 .map(|v| v.label())
7277 .collect::<Vec<_>>()
7278 .join(", ")
7279 ),
7280 );
7281 let panel: Vec<ReviewSeatReport<'_>> = records
7284 .iter()
7285 .filter_map(|r| {
7286 r.vote.map(|vote| ReviewSeatReport {
7287 reviewer: r.reviewer,
7288 vote,
7289 summary: &r.summary,
7290 findings: &r.findings,
7291 })
7292 })
7293 .collect();
7294
7295 let mut jobs = Vec::new();
7296 let mut seats_at = Vec::new();
7297 for (r, spec) in reviewers.iter().cloned().enumerate() {
7298 if records[r].vote.is_none() {
7302 continue;
7303 }
7304 let wt = root.join(format!("review-{}", r + 1));
7305 let seat_key = format!("review-{}", r + 1);
7306 let spec = self.occupant(&seat_key, spec);
7307 let seat = self.seat(&seat_key, &spec.id);
7308 let build = |with_patch: bool| {
7316 prompt::review_reconsider(&ReviewReconsiderCtx {
7317 instruction: &self.state.instruction,
7318 reviewer: r + 1,
7319 lens: Lens::for_seat(r),
7320 panel: &panel,
7321 patch: with_patch.then_some(ReviewPatch {
7322 branch: &winner.branch,
7323 base_short: &base_short,
7324 stat: &stat,
7325 patch: &patch,
7326 }),
7327 round,
7328 rounds: max_rounds,
7329 language: &language,
7330 })
7331 };
7332 let full = build(true);
7333 let prompt = if has_context(&spec, &seat, sessions) {
7334 build(false)
7335 } else {
7336 full.clone()
7337 };
7338 jobs.push(SeatJob {
7339 prompt,
7340 spec,
7341 seat,
7342 cwd: wt,
7343 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7344 allow_write: false,
7345 sessions,
7346 artifacts: artifacts.clone(),
7347 stem: format!("review-{round}-reconsider-{}", r + 1),
7348 handover: Some(full),
7349 });
7350 seats_at.push(r);
7351 }
7352
7353 let mut recon_quota_losses = Vec::new();
7354 let recon_cache = self.state.config.cache_dir();
7355 let recon_ctx = WaveCtx {
7356 carry_seats: true,
7357 pauses: Some((&self.pause, &self.interrupt)),
7358 run: &run_id,
7359 node: "review",
7360 prompts: &prompts,
7361 cache: recon_cache.as_deref(),
7362 round: Some(round),
7363 };
7364 let recon_results = ask_json_wave::<ReviewRevote>(
7365 jobs,
7366 Arc::clone(&self.sem),
7367 review_retries,
7368 &self.roles.reviewer_roster,
7369 &recon_ctx,
7370 &mut recon_quota_losses,
7371 &mut self.state,
7372 &|_: &ReviewRevote| Ok(()),
7373 )
7374 .await;
7375 if self.state.resource_block.is_some() {
7376 return self.stop_for_resource();
7377 }
7378 self.state.quota.extend(recon_quota_losses);
7379
7380 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7381 let agent_id = seat.agent.clone();
7382 self.state.seats.insert(seat.key.clone(), seat);
7383 let mut rec = ReviewRevoteRecord {
7384 reviewer: r + 1,
7385 agent: agent_id,
7386 vote: None,
7387 reason: String::new(),
7388 failed: None,
7389 };
7390 match res {
7391 Ok((rv, _)) => {
7392 rec.vote = Some(rv.vote);
7393 rec.reason =
7394 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7395 self.state.event(
7396 "review",
7397 format!(
7398 "round {round}: reviewer {} revoted {}",
7399 r + 1,
7400 rv.vote.label()
7401 ),
7402 );
7403 }
7404 Err(e) => {
7405 rec.failed = Some(e.to_string());
7406 self.state.event(
7407 "review",
7408 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7409 );
7410 }
7411 }
7412 reconsideration.push(rec);
7413 }
7414 } else if initial_votes.len() > 1 {
7415 self.state.event(
7416 "review",
7417 format!(
7418 "round {round}: votes agreed ({}) — no reconsideration",
7419 initial_votes[0].label()
7420 ),
7421 );
7422 }
7423
7424 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7425 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7426 let defer_e2e =
7437 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7438 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7439 let reason =
7440 format!("{blocking} blocking finding(s) already required a fix this round");
7441 self.state.event(
7442 "verify",
7443 format!(
7444 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7445 {}); it will run once a round has none left",
7446 short(&head)
7447 ),
7448 );
7449 (Vec::new(), false, true, Some(reason))
7450 } else {
7451 let e2e_commands = self.state.config.verify.e2e.clone();
7452 let cache_dir = self.state.config.cache_dir();
7453 let context = format!("round {round}");
7454 let wait = self.repository_wait();
7455 let (e2e, verify_retried) = with_cache_lease(
7456 &mut self.state,
7457 cache_dir.as_deref(),
7458 "e2e",
7459 "e2e",
7460 &winner.worktree,
7461 &head,
7462 verify_timeout,
7463 &context,
7464 &wait,
7465 |state, budget| {
7466 let shell = shell.clone();
7467 let e2e_commands = e2e_commands.clone();
7468 let worktree = winner.worktree.clone();
7469 let context = context.clone();
7470 async move {
7471 run_e2e_with_retry(
7472 state,
7473 &shell,
7474 &e2e_commands,
7475 &worktree,
7476 budget,
7477 &context,
7478 )
7479 .await
7480 }
7481 },
7482 )
7483 .await?;
7484 (e2e, verify_retried, false, None)
7485 };
7486
7487 let expected = records.len();
7488 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7489 let incomplete = answered < expected;
7490 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7491 let policy = self.state.config.graph.incomplete_review;
7492 let clean = round_is_clean(
7493 blocking,
7494 e2e_ok,
7495 answered,
7496 expected,
7497 round_quota_missing,
7498 policy,
7499 );
7500
7501 let mut round_record = ReviewRound {
7502 round,
7503 head: head.clone(),
7504 verified_head: None,
7505 verified_at: None,
7506 reviews: records,
7507 e2e,
7508 verify_retried,
7509 e2e_deferred,
7510 e2e_defer_reason,
7511 fix: None,
7512 blocking,
7513 answered,
7514 expected,
7515 clean,
7516 progressed: false,
7517 vote_split,
7518 reconsideration,
7519 verdict: None,
7520 };
7521 round_record.verdict = ReviewVote::worst(
7525 round_record
7526 .final_votes()
7527 .into_iter()
7528 .map(|(_, _, vote)| vote),
7529 );
7530 if !matches!(
7541 round_record.e2e_status(),
7542 E2eStatus::Deferred | E2eStatus::NotConfigured
7543 ) {
7544 round_record.verified_head = Some(head.clone());
7545 round_record.verified_at = Some(Timestamp::now());
7546 }
7547
7548 if incomplete {
7549 let missing: Vec<String> = round_record
7550 .reviews
7551 .iter()
7552 .filter(|r| r.failed.is_some())
7553 .map(|r| format!("review-{}", r.reviewer))
7554 .collect();
7555 self.state.event(
7556 "review",
7557 format!(
7558 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7559 missing.join(", ")
7560 ),
7561 );
7562 }
7563
7564 if clean {
7565 self.state.event(
7566 "review",
7567 if incomplete && policy == IncompleteReviewPolicy::Warn {
7568 format!(
7569 "round {round}: clean (warn policy, incomplete panel) — no \
7570 blocking findings from the seats that answered, verification green"
7571 )
7572 } else if incomplete {
7573 format!(
7574 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7575 quorum) — no blocking findings from the seats that answered, \
7576 verification green",
7577 expected - answered
7578 )
7579 } else {
7580 format!("round {round}: clean — no blocking findings, verification green")
7581 },
7582 );
7583 self.state.reviews.push(round_record);
7584 self.state.status = RunStatus::Gating;
7585 self.state.save()?;
7586 return Ok(());
7587 }
7588
7589 if incomplete && blocking == 0 && e2e_ok {
7597 self.state.reviews.push(round_record);
7598 self.state.save()?;
7599 if round == max_rounds {
7600 self.state.status = RunStatus::Blocked;
7601 self.state.event(
7602 "review",
7603 format!(
7604 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7605 refusing to call it clean",
7606 expected - answered
7607 ),
7608 );
7609 return Ok(());
7610 }
7611 continue;
7612 }
7613
7614 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7626 self.state.reviews.push(round_record);
7627 return self
7628 .stop_unresolved_review(
7629 "the round's own verification could not run",
7630 &shell,
7631 &winner.worktree,
7632 )
7633 .await;
7634 }
7635
7636 if round == max_rounds {
7637 self.state.reviews.push(round_record);
7638 return self
7639 .stop_reviewing(
7640 &format!(
7641 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7642 ),
7643 &shell,
7644 &winner.worktree,
7645 )
7646 .await;
7647 }
7648
7649 if !self.fix_review_round(&winner, round_record).await? {
7650 return Ok(());
7651 }
7652 }
7653 Ok(())
7654 }
7655
7656 async fn fix_review_round(
7659 &mut self,
7660 winner: &Candidate,
7661 mut round_record: ReviewRound,
7662 ) -> Result<bool> {
7663 let round = round_record.round;
7664 let shell = self.state.config.shell();
7665 let step = self.run_fix_round(winner, &mut round_record).await?;
7666 if matches!(step, FixStep::Resource) {
7667 self.stop_for_resource()?;
7668 return Ok(false);
7669 }
7670 if self.state.reviews.len() < round {
7671 self.state.reviews.push(round_record);
7672 } else {
7673 self.state.reviews[round - 1] = round_record;
7674 }
7675 self.state.save()?;
7676
7677 if matches!(step, FixStep::Recorded { report_lost: true }) {
7690 self.stop_reviewing(
7691 "the fixer's adoption report never came back, even after resuming its \
7692 own seat; refusing to start another round against the same worktree \
7693 while that is unresolved",
7694 &shell,
7695 &winner.worktree,
7696 )
7697 .await?;
7698 return Ok(false);
7699 }
7700
7701 let streak = self
7702 .state
7703 .reviews
7704 .iter()
7705 .rev()
7706 .take_while(|r| !r.progressed)
7707 .count();
7708 if streak >= STAGNANT_LIMIT {
7709 self.stop_reviewing(
7710 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7711 &shell,
7712 &winner.worktree,
7713 )
7714 .await?;
7715 return Ok(false);
7716 }
7717 Ok(true)
7718 }
7719
7720 async fn run_fix_round(
7722 &mut self,
7723 winner: &Candidate,
7724 round_record: &mut ReviewRound,
7725 ) -> Result<FixStep> {
7726 let round = round_record.round;
7727 let max_rounds = self.state.config.graph.review_rounds;
7728 let run_id = self.state.id.clone();
7729 let prompts = self.state.config.prompts.clone();
7730 let language = self.state.config.graph.language.clone();
7731 let sessions = self.state.config.graph.sessions;
7732 let artifacts = self.merge_repair_artifacts();
7733 let base = self.landing_base();
7734 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7735 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7736 let this_round_verification = round_record.verification_summary(&head);
7737 let all_findings: Vec<_> = round_record
7738 .reviews
7739 .iter()
7740 .flat_map(|r| r.findings.iter().cloned())
7741 .collect();
7742 let blocking_findings: Vec<_> = all_findings
7745 .iter()
7746 .filter(|f| f.severity.blocks())
7747 .cloned()
7748 .collect();
7749 let fix_prompt = prompt::fix_in(
7750 &self.state.instruction,
7751 &blocking_findings,
7752 this_round_verification.as_ref(),
7753 round,
7754 max_rounds,
7755 &language,
7756 &self.state.config.graph.github_language,
7757 );
7758 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7759 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7760 let (job, seat, out) = self
7761 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7762 prompt: fix_prompt.clone(),
7763 spec,
7764 seat,
7765 cwd: winner.worktree.clone(),
7766 timeout,
7767 allow_write: true,
7768 sessions,
7769 artifacts: artifacts.clone(),
7770 stem: format!("fix-{round}"),
7771 handover: None,
7772 })
7773 .await;
7774 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7777 return Ok(FixStep::Resource);
7778 }
7779 let agent_id = seat.agent.clone();
7780
7781 let mut fix = FixRecord {
7782 agent: agent_id,
7783 addressed: Vec::new(),
7784 rejected: Vec::new(),
7785 notes: String::new(),
7786 committed: false,
7787 failed: None,
7788 duration_ms: 0,
7789 continuation: None,
7790 };
7791 let mut continuation = ContinuationRecord::not_needed();
7792 let mut final_seat = seat.clone();
7793 match out {
7794 AgentOutcome::Ok(o) => {
7795 fix.duration_ms = o.duration_ms;
7796 let parsed = verdict::extract_json::<FixReport>(&o.text);
7797 let incomplete_reason = match &parsed {
7804 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7805 "the reply parsed, but it reported a command whose own CLI never \
7806 confirmed an exit status"
7807 .to_owned(),
7808 ),
7809 Ok(_) => None,
7810 Err(e) => Some(e.to_string()),
7811 };
7812 match incomplete_reason {
7813 None => {
7814 let report = parsed.expect("checked Ok above");
7815 fix.addressed = report.addressed;
7816 fix.rejected = report.rejected;
7817 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7818 }
7819 Some(reason) => {
7820 let (resumed_seat, resolved, failure, cont) = self
7821 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7822 .await;
7823 fix.duration_ms += cont.cumulative_wait_ms;
7824 continuation = cont;
7825 final_seat = resumed_seat;
7826 match resolved {
7827 Some(report) => {
7828 fix.addressed = report.addressed;
7829 fix.rejected = report.rejected;
7830 fix.notes =
7831 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7832 }
7833 None => fix.failed = failure,
7834 }
7835 }
7836 }
7837 }
7838 AgentOutcome::Dropped(o) => {
7840 fix.duration_ms = o.duration_ms;
7841 let why = o
7842 .dropped
7843 .as_ref()
7844 .map(|d| d.why.as_str())
7845 .unwrap_or("the CLI ended the stream without delivering its answer");
7846 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7847 }
7848 AgentOutcome::Quota(o) => {
7849 self.state.quota.push(QuotaLoss::from_report(
7850 final_seat.key.clone(),
7851 "fix",
7852 o.quota.as_ref(),
7853 ));
7854 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7855 }
7856 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7857 }
7858 if self.state.resource_block.is_some() {
7859 self.state.seats.insert(final_seat.key.clone(), final_seat);
7860 return Ok(FixStep::Resource);
7861 }
7862 fix.continuation = Some(continuation);
7863 self.state.seats.insert(final_seat.key.clone(), final_seat);
7864 let coordination = self.coordinate(&winner.worktree).await?;
7865 if let Ok(r) = git::rescue_commit(
7866 &winner.worktree,
7867 &format!("magi: review round {round} fixes (uncommitted work)"),
7868 )
7869 .await
7870 {
7871 self.state.note_withheld("fix", &r.withheld);
7872 }
7873 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7874 fix.committed = after != before;
7875 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7883 drop(coordination);
7884 let progressed = diff_after != patch;
7885 let commit_note = if fix.committed {
7886 "committed"
7887 } else {
7888 "NO new commit"
7889 };
7890 let tree_note = if progressed {
7891 "changed vs base"
7892 } else {
7893 "unchanged vs base"
7894 };
7895 self.state.event(
7896 "fix",
7897 match &fix.failed {
7898 Some(reason) => {
7904 format!(
7905 "round {round}: fixer's adoption report was lost ({reason}); \
7906 {commit_note}, tree {tree_note}"
7907 )
7908 }
7909 None => format!(
7910 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7911 {tree_note}{}",
7912 fix.addressed.len(),
7913 fix.rejected.len(),
7914 if continuation.outcome == ContinuationOutcome::Resumed {
7915 format!(
7916 " (adoption report recovered after {} continuation(s))",
7917 continuation.attempts
7918 )
7919 } else {
7920 String::new()
7921 },
7922 ),
7923 },
7924 );
7925 round_record.fix = Some(fix);
7926 round_record.progressed = progressed;
7927 Ok(FixStep::Recorded {
7928 report_lost: matches!(
7929 continuation.outcome,
7930 ContinuationOutcome::Exhausted
7931 | ContinuationOutcome::QuotaLost
7932 | ContinuationOutcome::NoSession
7933 ),
7934 })
7935 }
7936
7937 fn record_contested_handoff(&mut self) {
7971 if self.state.contested_handoff.is_some() {
7972 return;
7973 }
7974 let Some(contested) = self
7975 .state
7976 .reviews
7977 .last()
7978 .and_then(ReviewRound::contested_handoff)
7979 else {
7980 return;
7981 };
7982 self.state.event(
7983 "review",
7984 format!(
7985 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7986 wait for the owner's approval",
7987 contested.findings.len(),
7988 contested.rejecters.len()
7989 ),
7990 );
7991 self.state.contested_handoff = Some(contested);
7992 }
7993
7994 async fn stop_unresolved_review(
7995 &mut self,
7996 why: &str,
7997 shell: &[String],
7998 worktree: &Path,
7999 ) -> Result<()> {
8000 self.stop_reviewing_inner(why, shell, worktree, true).await
8001 }
8002
8003 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8004 self.stop_reviewing_inner(why, shell, worktree, false).await
8005 }
8006
8007 async fn stop_reviewing_inner(
8008 &mut self,
8009 why: &str,
8010 shell: &[String],
8011 worktree: &Path,
8012 repairable: bool,
8013 ) -> Result<()> {
8014 let round_idx = self.state.reviews.len() - 1;
8015 let needs_catchup_run = matches!(
8023 self.state.reviews[round_idx].e2e_status(),
8024 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8025 );
8026 if needs_catchup_run {
8027 let round = self.state.reviews[round_idx].round;
8028 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8029 let commands = self.state.config.verify.e2e.clone();
8030 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8031 let cache_dir = self.state.config.cache_dir();
8032 let context = format!(
8033 "round {round}: verification unresolved, catching up before the final decision"
8034 );
8035 let wait = self.repository_wait();
8036 let (outcomes, verify_retried) = with_cache_lease(
8037 &mut self.state,
8038 cache_dir.as_deref(),
8039 "e2e",
8040 "e2e",
8041 worktree,
8042 &attempted_head,
8043 timeout,
8044 &context,
8045 &wait,
8046 |state, budget| {
8047 let shell = shell.to_vec();
8048 let commands = commands.clone();
8049 let context = context.clone();
8050 async move {
8051 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8052 .await
8053 }
8054 },
8055 )
8056 .await?;
8057 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8058 self.state.record_base_progress(&attempted_head);
8059 }
8060 let last = &mut self.state.reviews[round_idx];
8061 last.e2e = outcomes;
8062 last.verify_retried = verify_retried;
8063 last.verified_head = Some(attempted_head);
8070 last.verified_at = Some(Timestamp::now());
8071 if verify_inconclusive(&last.e2e) {
8072 self.state.save()?;
8079 return Ok(());
8080 }
8081 last.e2e_deferred = false;
8082 if last.answered == last.expected {
8087 last.clean = round_is_clean(
8088 last.blocking,
8089 last.e2e.iter().all(CommandOutcome::ok),
8090 last.answered,
8091 last.expected,
8092 0,
8093 self.state.config.graph.incomplete_review,
8094 );
8095 }
8096 }
8097 let last = &self.state.reviews[round_idx];
8098 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8099
8100 match last.e2e_status() {
8101 E2eStatus::Failed => {
8102 let red: Vec<String> = last
8103 .e2e
8104 .iter()
8105 .filter(|o| !o.ok())
8106 .map(|o| {
8107 format!(
8108 "`{}` -> {:?}\n{}",
8109 o.command,
8110 o.code,
8111 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8112 )
8113 })
8114 .collect();
8115 let can_repair = last.blocking == 0 && last.fix.is_none();
8116 self.state.reviews[round_idx].clean = false;
8117 self.state
8118 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8119 self.state.status = RunStatus::Blocked;
8120 if repairable
8121 && can_repair
8122 && self.state.reviews.len() < self.state.config.graph.review_rounds
8123 {
8124 self.state.status = RunStatus::Reviewing;
8125 }
8126 }
8127 E2eStatus::ResourceBlocked => {
8132 self.state.event(
8133 "review",
8134 format!(
8135 "{why}; e2e could not run (resource unavailable); not \
8136 deciding yet"
8137 ),
8138 );
8139 }
8140 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8141 self.state.event(
8142 "review",
8143 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8144 );
8145 self.record_contested_handoff();
8146 self.state.status = RunStatus::Gating;
8147 }
8148 }
8149 self.state.save()?;
8150 Ok(())
8151 }
8152
8153 async fn gate(&mut self) -> Result<()> {
8156 if self.hold_base_sync().await? {
8157 return Ok(());
8158 }
8159 if self.state.status == RunStatus::Failed
8171 || self
8172 .state
8173 .base_sync
8174 .as_ref()
8175 .is_some_and(|s| s.conflict.is_some())
8176 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8177 != Some(RunStatus::Gating)
8178 {
8179 return Ok(());
8180 }
8181 let Some(winner) = self.state.winner().cloned() else {
8182 return Ok(());
8183 };
8184 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8185 self.state.reconcile_verification_head(&head);
8186 self.state.save()?;
8187 let wait = self.repository_wait();
8188 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8189 return Ok(());
8190 }
8191 if self.state.gate_ran {
8192 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8203 self.state.status = RunStatus::Blocked;
8204 self.state.save()?;
8205 }
8206 return Ok(());
8207 }
8208 self.state.status = RunStatus::Gating;
8209 let mut outcomes = self.run_gate(&winner).await?;
8210 let mut attempted_head = self.state.gate_head.clone();
8211 loop {
8212 if verify_inconclusive(&outcomes) {
8223 self.state.save()?;
8224 return Ok(());
8225 }
8226 if outcomes.iter().all(CommandOutcome::ok) {
8227 break;
8228 }
8229 let fixes_before = self.state.gate_fixes.len();
8230 match self.gate_fix_round(&winner, &outcomes).await? {
8231 GateFix::Retry => {
8232 outcomes = self.run_gate(&winner).await?;
8233 attempted_head = self.state.gate_head.clone();
8234 }
8235 GateFix::Stop => {
8236 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8237 if self.state.gate_fixes.len() > fixes_before
8238 && attempted_head.as_deref() != Some(current.as_str())
8239 && verify_current_head(&mut self.state, &winner, &wait).await?
8240 {
8241 outcomes = self.run_gate(&winner).await?;
8244 attempted_head = self.state.gate_head.clone();
8245 } else {
8246 break;
8247 }
8248 }
8249 GateFix::Defer => {
8250 self.state.save()?;
8251 return Ok(());
8252 }
8253 }
8254 }
8255 let passed = outcomes.iter().all(CommandOutcome::ok);
8256 self.state.gate = outcomes;
8257 self.state.gate_ran = true;
8258 self.state.gate_head = attempted_head;
8259 if !self.state.gate.is_empty()
8260 && !verify_inconclusive(&self.state.gate)
8261 && let Some(head) = self.state.gate_head.clone()
8262 {
8263 self.state.record_base_progress(&head);
8264 }
8265 self.state.save()?;
8266 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8267 return Ok(());
8268 }
8269 if !passed {
8270 self.state.status = RunStatus::Blocked;
8271 let spent = self.state.gate_fixes.len();
8272 self.state.event(
8273 "gate",
8274 if spent == 0 {
8275 "gate failed; not merging".to_owned()
8276 } else {
8277 format!("gate failed after {spent} gate-fix round(s); not merging")
8278 },
8279 );
8280 }
8281 self.state.save()?;
8282 Ok(())
8283 }
8284
8285 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8295 let commands = state.config.verify.pre_gate.clone();
8296 if commands.is_empty() {
8297 return;
8298 }
8299 let shell = state.config.shell();
8300 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8301 let (outcomes, _) = run_commands(
8302 state,
8303 "pre_gate",
8304 "pre_gate",
8305 0,
8306 &shell,
8307 &commands,
8308 &winner.worktree,
8309 timeout,
8310 )
8311 .await;
8312 for o in &outcomes {
8313 if !o.ok() {
8314 tracing::warn!(
8315 "pre_gate `{}` failed ({:?}); the gate decides",
8316 o.command,
8317 o.code
8318 );
8319 }
8320 state.event(
8321 "pre_gate",
8322 format!(
8323 "`{}` -> {}",
8324 o.command,
8325 if o.ok() {
8326 "pass".to_owned()
8327 } else {
8328 format!(
8329 "FAIL ({:?})\n{}",
8330 o.code,
8331 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8332 )
8333 }
8334 ),
8335 );
8336 }
8337 state.pre_gate = outcomes;
8338 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8339 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8340 Ok(head) => {
8341 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8342 state.pre_gate_commit = Some(head);
8343 }
8344 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8345 },
8346 Ok(false) => {}
8347 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8348 }
8349 if let Err(e) = state.save() {
8350 tracing::warn!("could not persist the pre_gate record: {e:#}");
8351 }
8352 }
8353
8354 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8357 let wait = self.repository_wait();
8358 run_landing_gate(&mut self.state, winner, &wait).await
8359 }
8360
8361 async fn gate_fix_round(
8374 &mut self,
8375 winner: &Candidate,
8376 outcomes: &[CommandOutcome],
8377 ) -> Result<GateFix> {
8378 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8379 g.phase == crate::repair_budget::Phase::Fixing
8380 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8381 }) {
8382 self.state.gate_fixes.len().saturating_add(1)
8383 } else {
8384 self.state.config.graph.gate_fix_rounds
8385 };
8386 let spent = self.state.gate_fixes.len();
8387 if spent >= cap {
8388 if cap > 0 {
8389 self.state.event(
8390 "gate",
8391 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8392 );
8393 }
8394 return Ok(GateFix::Stop);
8395 }
8396 if !gate_fixable(outcomes) {
8397 self.state.event(
8398 "gate",
8399 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8400 command or similar); not spending a fix round on it",
8401 );
8402 return Ok(GateFix::Stop);
8403 }
8404 let min_free = self.state.config.disk.min_free_bytes;
8405 if min_free > 0 {
8406 match crate::disk::free_bytes(&winner.worktree) {
8407 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8408 Ok(free) => {
8409 self.state.event(
8410 "gate",
8411 format!(
8412 "only {free} bytes free ({min_free} required by `[disk] \
8413 min_free_bytes`); not spending a fix round on a failure the disk \
8414 may explain"
8415 ),
8416 );
8417 return Ok(GateFix::Stop);
8418 }
8419 Err(e) => {
8420 self.state.event(
8421 "gate",
8422 format!("free disk space could not be measured ({e:#}); no fix round"),
8423 );
8424 return Ok(GateFix::Stop);
8425 }
8426 }
8427 }
8428
8429 let attempt = spent + 1;
8430 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8431 let base = self.landing_base();
8432 let fix_prompt = prompt::gate_fix_in(
8433 &self.state.instruction,
8434 &failed,
8435 attempt,
8436 cap,
8437 &self.state.config.graph.language,
8438 &self.state.config.graph.github_language,
8439 );
8440 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8441 let sessions = self.state.config.graph.sessions;
8442 let artifacts = self.merge_repair_artifacts();
8443 self.state.event(
8444 "gate",
8445 format!("gate failed; gate-fix round {attempt} of {cap}"),
8446 );
8447 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8448 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8449 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8452 head: before.clone(),
8453 passed: false,
8454 outcomes: Vec::new(),
8455 });
8456 self.state.save()?;
8457 let (_, seat, out) = self
8458 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8459 prompt: fix_prompt.clone(),
8460 spec,
8461 seat,
8462 cwd: winner.worktree.clone(),
8463 timeout,
8464 allow_write: true,
8465 sessions,
8466 artifacts: artifacts.clone(),
8467 stem: format!("gate-fix-{attempt}"),
8468 handover: None,
8469 })
8470 .await;
8471 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8474 return Ok(GateFix::Defer);
8475 }
8476 let mut record = GateFixRecord {
8477 agent: seat.agent.clone(),
8478 failed,
8479 notes: String::new(),
8480 committed: false,
8481 error: None,
8482 };
8483 match out {
8484 AgentOutcome::Ok(o) => {
8485 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8488 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8489 }
8490 }
8491 AgentOutcome::Dropped(_) => {
8492 record.error = Some("the CLI dropped the stream".to_owned());
8493 }
8494 AgentOutcome::Quota(o) => {
8495 self.state.quota.push(QuotaLoss::from_report(
8496 seat.key.clone(),
8497 "gate-fix",
8498 o.quota.as_ref(),
8499 ));
8500 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8501 }
8502 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8503 }
8504 self.state.seats.insert(seat.key.clone(), seat);
8505 let coordination = self.coordinate(&winner.worktree).await?;
8506 if let Ok(r) = git::rescue_commit(
8507 &winner.worktree,
8508 &format!("magi: gate fix {attempt} (uncommitted work)"),
8509 )
8510 .await
8511 {
8512 self.state.note_withheld("gate-fix", &r.withheld);
8513 }
8514 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8515 record.committed = after != before;
8516 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8517 let note = record.error.clone();
8518 self.state.gate_fixes.push(record);
8519 self.state.save()?;
8520 if !changed {
8521 self.state.event(
8522 "gate",
8523 match note {
8524 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8525 None => format!("gate-fix round {attempt}: the tree did not change"),
8526 },
8527 );
8528 return Ok(GateFix::Stop);
8529 }
8530 self.state.event(
8531 "gate",
8532 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8533 );
8534
8535 drop(coordination);
8536 let wait = self.repository_wait();
8537 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8538 }
8539
8540 async fn guarded_pr_message(
8547 &mut self,
8548 winner: &Candidate,
8549 facts: Option<&BranchFacts>,
8550 posting: bool,
8551 ) -> Result<Option<PrMessage>> {
8552 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8553 let head = match final_head.as_deref() {
8554 Some(h) => crate::pr_outcome::FinalHead::At(h),
8555 None => crate::pr_outcome::FinalHead::Unknown,
8556 };
8557 let summarized = if posting {
8559 Some(self.summarize_change(winner, final_head.as_deref()).await)
8560 } else {
8561 None
8562 };
8563 if self.state.resource_block.is_some() {
8568 self.state.event(
8569 "pr-summary",
8570 "a shared resource was busy; nothing was published, resume to summarize again",
8571 );
8572 return Ok(None);
8573 }
8574 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8575 anyhow::bail!(
8576 "the final commit moved during public description generation; retry before publishing"
8577 );
8578 }
8579 let note = match (&summarized, &self.state.change_summary) {
8580 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8581 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8582 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8583 (None, _) => ChangeNote::NotAsked,
8584 };
8585 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8586 if !posting {
8587 let identity = crate::scrub::Identity::current();
8588 return Ok(Some(PrMessage {
8589 title: crate::scrub::scrub(&pr.title, &identity),
8590 body: crate::scrub::scrub(&pr.body, &identity),
8591 }));
8592 }
8593 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8596 if let Some(known) = self
8597 .state
8598 .github_text
8599 .clone()
8600 .filter(|g| g.fingerprint == raw_fp)
8601 {
8602 return self.settle_withheld(winner, &pr, known).await;
8603 }
8604 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8605 let mut violations =
8606 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8607 let identity = crate::scrub::Identity::current();
8608 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8609 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8610 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8611 {
8612 violations.push(crate::github_text::Violation::SensitiveData);
8613 }
8614 let has_language = violations
8618 .iter()
8619 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8620 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8621 self.state.event(
8622 "github-text",
8623 if has_language {
8624 format!("description rejected: {violations:?}; requesting one rewrite")
8625 } else {
8626 format!("description rejected: {violations:?}; asking the owner")
8627 },
8628 );
8629 let mut rewritten = false;
8630 let mut rejected_rewrite = None;
8631 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8632 {
8633 let output = text;
8634 if let Some(title) = summary_title(&output) {
8635 let raw_body = summary_without_title(&output);
8638 let clean_title = crate::scrub::scrub(&title, &identity);
8639 let mut body = crate::scrub::scrub(&raw_body, &identity);
8640 let redacted = clean_title != title || body != raw_body;
8641 let title = clean_title;
8642 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8643 if !complete {
8644 self.state.event(
8645 "github-text",
8646 "description rewrite omitted required findings",
8647 );
8648 }
8649 if !body.trim().is_empty() {
8650 body.push_str("\n\n");
8651 body.push_str(&public_pr_evidence(&self.state, head));
8652 }
8653 let marker = format!("magi:run/{}", self.state.id);
8654 if !body.trim().is_empty() && !body.contains(&marker) {
8655 body.push_str(&format!("\n\n{marker}"));
8656 }
8657 let rewrite = PrMessage {
8658 title: title.clone(),
8659 body: body.clone(),
8660 };
8661 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8662 let mut failed = crate::github_text::check_for(
8663 &self.state.config.graph,
8664 &title,
8665 &body,
8666 redecision,
8667 );
8668 if (!crate::github_text::shareable(&title)
8669 || !crate::github_text::shareable(&body))
8670 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8671 {
8672 failed.push(crate::github_text::Violation::SensitiveData);
8673 }
8674 if complete
8675 && !title.trim().is_empty()
8676 && !body.trim().is_empty()
8677 && !failed.is_empty()
8678 {
8679 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8680 title: title.clone(),
8681 body: body.clone(),
8682 failed: failed
8683 .iter()
8684 .map(|v| crate::github_text::category(*v).to_owned())
8685 .collect(),
8686 });
8687 }
8688 if complete
8689 && !title.trim().is_empty()
8690 && !raw_body.trim().is_empty()
8691 && failed.is_empty()
8692 {
8693 decision = redecision;
8694 pr = PrMessage { title, body };
8695 rewritten = true;
8696 self.state
8697 .event("github-text", "description rewrite passed");
8698 if redacted {
8699 self.state.event(
8700 "github-text",
8701 "sensitive data removed from description rewrite",
8702 );
8703 }
8704 }
8705 }
8706 }
8707 if !rewritten {
8708 self.state.event(
8709 "github-text",
8710 "description rewrite unavailable or rejected; asking the owner",
8711 );
8712 if let Some(w) =
8713 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8714 {
8715 self.retire_github_text_question("the withheld text changed");
8718 let record = crate::run::GithubTextAsk {
8719 rewrite: rejected_rewrite,
8720 fingerprint: raw_fp,
8721 title: w.title,
8722 body: w.body,
8723 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8724 question: None,
8725 asks: 0,
8726 resolved: false,
8727 chosen_title: None,
8728 choice: None,
8729 };
8730 return self.settle_withheld(winner, &pr, record).await;
8731 }
8732 }
8733 }
8734 let (title, body) =
8735 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8736 Ok(Some(PrMessage { title, body }))
8737 }
8738
8739 async fn summarize_change(
8750 &mut self,
8751 winner: &Candidate,
8752 head: Option<&str>,
8753 ) -> std::result::Result<(), String> {
8754 let Some(head) = head else {
8755 return Err("the final commit could not be read".to_owned());
8756 };
8757 let remote = self.state.config.merge.remote.clone();
8758 let tracking = format!("{remote}/{}", self.state.base_branch);
8759 if !git::rev_exists(&winner.worktree, &tracking).await {
8760 return Err(format!("the remote base `{tracking}` could not be read"));
8761 }
8762 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8763 Ok(b) if !b.is_empty() => b,
8764 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8765 };
8766 let evidence = crate::pr_outcome::summary_evidence(
8767 &self.state,
8768 crate::pr_outcome::FinalHead::At(head),
8769 );
8770 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8771 if self
8772 .state
8773 .change_summary
8774 .as_ref()
8775 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8776 {
8777 return Ok(());
8778 }
8779 if self.state.change_summary.take().is_some() {
8780 self.state.event(
8781 "pr-summary",
8782 "the final diff or evidence changed; the earlier change summary was discarded",
8783 );
8784 }
8785 if let Err(e) = write_artifact(
8787 &self.state,
8788 &format!("public-description-{fingerprint}.md"),
8789 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8790 ) {
8791 tracing::warn!("could not save public description evidence: {e:#}");
8792 }
8793 let chain = match agent::pick_chain(
8794 &self.state.config.agents,
8795 self.state.config.roles.synthesizer.as_ref(),
8796 &agent::installed,
8797 "synthesizer",
8798 ) {
8799 Ok(chain) => chain,
8800 Err(e) => {
8801 tracing::warn!("no agent for the change summary: {e:#}");
8802 self.state.event("pr-summary", "no summary agent available");
8803 return Err("no agent was available to summarize it".to_owned());
8804 }
8805 };
8806 let subjects = git::subjects(&winner.worktree, &base, head)
8807 .await
8808 .unwrap_or_default();
8809 let stat = git::diff_stat(&winner.worktree, &base, head)
8810 .await
8811 .unwrap_or_default();
8812 let stat: String = stat.chars().take(6_000).collect();
8813 let mut prompt = crate::prompt::change_summary_in(
8814 &base,
8815 head,
8816 &subjects,
8817 stat.trim(),
8818 &self.state.config.graph.github_language,
8819 );
8820 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8821 let sessions = self.state.config.graph.sessions;
8822 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8823 let prompts = self.state.config.prompts.clone();
8824 let cache = self.state.config.cache_dir();
8825 let run = self.state.id.clone();
8826 for (i, spec) in chain.into_iter().enumerate() {
8827 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8828 let github_spec = match agent::for_github_text(
8829 &spec,
8830 &self.state.config.graph.github_language,
8831 &winner.worktree,
8832 ) {
8833 Ok(spec) => spec,
8834 Err(e) => {
8835 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8836 continue;
8837 }
8838 };
8839 let job = SeatJob {
8840 spec: github_spec,
8841 seat,
8842 cwd: winner.worktree.clone(),
8843 prompt: prompt.clone(),
8844 timeout,
8845 allow_write: false,
8846 sessions,
8847 artifacts: agent::artifacts_dir(&self.state.dir()),
8848 stem: if i == 0 {
8849 "change-summary".to_owned()
8850 } else {
8851 format!("change-summary-{}", spec.id)
8852 },
8853 handover: None,
8854 };
8855 let ctx = WaveCtx {
8856 run: &run,
8857 node: "pr-summary",
8858 prompts: &prompts,
8859 cache: cache.as_deref(),
8860 round: None,
8861 carry_seats: false,
8862 pauses: Some((&self.pause, &self.interrupt)),
8863 };
8864 let (seat, outcome) =
8865 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8866 self.state.seats.insert(seat.key.clone(), seat);
8867 let text = match outcome {
8868 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8869 AgentOutcome::ResourceBlocked(why) => {
8870 if self.state.resource_block.is_none() {
8873 self.state.resource_block = Some(crate::run::ResourceBlock {
8874 cause: crate::run::ResourceBlockCause::Other,
8875 node: "pr-summary".to_owned(),
8876 kind: "shared resource".to_owned(),
8877 detail: why,
8878 at: Timestamp::now(),
8879 });
8880 }
8881 return Err("a shared resource was busy".to_owned());
8882 }
8883 _ => continue,
8884 };
8885 let blind = &self.state.config.blind;
8886 let Some(title) = summary_title(&text) else {
8887 continue;
8888 };
8889 let body: String = summary_without_title(&text)
8890 .lines()
8891 .filter(|l| !l.contains("magi:run/"))
8892 .collect::<Vec<_>>()
8893 .join("\n");
8894 let body = blind::sanitize_prose(body.trim(), blind);
8895 let title = blind::sanitize_prose(&title, blind);
8896 if body.trim().is_empty() || title.trim().is_empty() {
8897 continue;
8898 }
8899 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8900 continue;
8901 }
8902 if git::rev_parse(&winner.worktree, "HEAD")
8904 .await
8905 .ok()
8906 .as_deref()
8907 != Some(head)
8908 {
8909 return Err("the branch moved while the summary was written".to_owned());
8910 }
8911 if git::merge_base(&winner.worktree, &tracking, head)
8912 .await
8913 .ok()
8914 .as_deref()
8915 != Some(base.as_str())
8916 || crate::pr_outcome::summary_evidence(
8917 &self.state,
8918 crate::pr_outcome::FinalHead::At(head),
8919 ) != evidence
8920 {
8921 return Err("the summary evidence changed while it was written".to_owned());
8922 }
8923 self.state.change_summary = Some(crate::run::ChangeSummary {
8924 head: head.to_owned(),
8925 base,
8926 evidence: fingerprint,
8927 title,
8928 body,
8929 });
8930 self.state.event(
8931 "pr-summary",
8932 format!("change summary written by {} for the final diff", spec.id),
8933 );
8934 if let Err(e) = self.state.save() {
8935 tracing::warn!("could not save the change summary: {e:#}");
8936 }
8937 return Ok(());
8938 }
8939 self.state
8940 .event("pr-summary", "no agent produced a change summary");
8941 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8942 }
8943
8944 async fn rewrite_pr_text(
8954 &mut self,
8955 winner: &Candidate,
8956 pr: &PrMessage,
8957 violations: &[crate::github_text::Violation],
8958 ) -> Option<String> {
8959 let prompt = format!(
8960 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8961 pr.title, pr.body
8962 );
8963 let sessions = self.state.config.graph.sessions;
8964 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8965 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8968 let key = format!("impl-{}", winner.label);
8969 if let Some(spec) = self
8970 .state
8971 .config
8972 .agents
8973 .iter()
8974 .find(|a| a.id == winner.agent)
8975 .cloned()
8976 && let Some(existing) = self.state.seats.get(&key)
8977 && existing.agent == spec.id
8978 && agent::has_session(spec.kind, existing, sessions)
8979 {
8980 attempts.push((spec, Some(existing.clone())));
8981 }
8982 match agent::pick_chain(
8983 &self.state.config.agents,
8984 self.state.config.roles.language_judge.as_ref(),
8985 &agent::installed,
8986 "language_judge",
8987 ) {
8988 Ok(chain) => {
8989 for spec in chain {
8990 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8991 continue;
8992 }
8993 attempts.push((spec, None));
8994 }
8995 }
8996 Err(e) if attempts.is_empty() => {
8997 tracing::warn!("no agent for the description rewrite: {e:#}");
8998 self.state
8999 .event("github-text", "no rewrite agent available");
9000 return None;
9001 }
9002 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9003 }
9004 let prompts = self.state.config.prompts.clone();
9005 let cache = self.state.config.cache_dir();
9006 let run = self.state.id.clone();
9007 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9008 let resumed = resumed_seat.is_some();
9009 let seat = resumed_seat.unwrap_or_else(|| {
9010 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9011 });
9012 let github_spec = match agent::for_github_text(
9013 &spec,
9014 &self.state.config.graph.github_language,
9015 &winner.worktree,
9016 ) {
9017 Ok(spec) => spec,
9018 Err(e) => {
9019 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9020 continue;
9021 }
9022 };
9023 let job = SeatJob {
9024 spec: github_spec,
9025 seat,
9026 cwd: winner.worktree.clone(),
9027 prompt: prompt.clone(),
9028 timeout: retry_budget(timeout_full, resumed),
9029 allow_write: false,
9030 sessions,
9031 artifacts: agent::artifacts_dir(&self.state.dir()),
9032 stem: if i == 0 {
9033 "github-text-rewrite".to_owned()
9034 } else {
9035 format!("github-text-rewrite-{}", spec.id)
9036 },
9037 handover: None,
9038 };
9039 let ctx = WaveCtx {
9040 run: &run,
9041 node: "github-text",
9042 prompts: &prompts,
9043 cache: cache.as_deref(),
9044 round: None,
9045 carry_seats: false,
9046 pauses: Some((&self.pause, &self.interrupt)),
9047 };
9048 let (seat, outcome) =
9049 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9050 self.state.seats.insert(seat.key.clone(), seat);
9051 match outcome {
9052 AgentOutcome::Ok(o)
9053 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9054 {
9055 return Some(o.text);
9056 }
9057 AgentOutcome::ResourceBlocked(_) => return None,
9058 _ => {}
9059 }
9060 }
9061 None
9062 }
9063
9064 fn retire_github_text_question(&mut self, why: &str) {
9066 let Some(id) = self
9067 .state
9068 .github_text
9069 .as_ref()
9070 .and_then(|g| g.question.clone())
9071 else {
9072 return;
9073 };
9074 let _ = ask::Questions::open().update(&id, |q| {
9075 q.abandon(why);
9076 Ok(())
9077 });
9078 }
9079
9080 async fn settle_withheld(
9084 &mut self,
9085 winner: &Candidate,
9086 pr: &PrMessage,
9087 mut g: crate::run::GithubTextAsk,
9088 ) -> Result<Option<PrMessage>> {
9089 use crate::github_text as gt;
9090 let store = ask::Questions::open();
9091 let timeout = self.state.config.graph.answer_timeout;
9092 if !g.resolved {
9093 let mut standing = g
9094 .question
9095 .as_ref()
9096 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9097 if let Some(q) = standing.as_ref()
9098 && q.status.open()
9099 && gt::expired(q, timeout)
9100 {
9101 let why = format!("no answer within {}s of asking", timeout.max(1));
9102 standing = store
9103 .update(&q.id, |q| {
9104 q.abandon(&why);
9105 Ok(())
9106 })
9107 .ok()
9108 .map(|(q, ())| q);
9109 }
9110 let mut ask_again: Option<Option<Vec<String>>> = None;
9111 match standing {
9112 None if g.asks <= 1 => {
9115 g.asks = 0;
9116 ask_again = Some(None);
9117 }
9118 None => {
9119 g.resolved = true;
9120 g.chosen_title = None;
9121 g.choice = Some(GithubTextChoice::Fallback);
9122 }
9123 Some(q) if q.status.open() => {
9124 self.park_on_github_text(&g)?;
9125 return Ok(None);
9126 }
9127 Some(q) => match gt::read_reply(&q) {
9128 gt::Reply::Fallback => {
9129 g.resolved = true;
9130 g.chosen_title = None;
9131 g.choice = Some(GithubTextChoice::Fallback);
9132 }
9133 gt::Reply::Original => {
9134 g.resolved = true;
9135 g.chosen_title = None;
9136 g.choice = Some(GithubTextChoice::Original);
9137 }
9138 gt::Reply::Rewrite => {
9139 g.resolved = true;
9140 g.chosen_title = None;
9141 g.choice = Some(if g.rewrite.is_some() {
9142 GithubTextChoice::Rewrite
9143 } else {
9144 GithubTextChoice::Fallback
9145 });
9146 }
9147 gt::Reply::Title(text) => {
9148 let first = text
9149 .lines()
9150 .map(str::trim)
9151 .find(|l| !l.is_empty())
9152 .unwrap_or("");
9153 let decision = if first.is_empty() {
9154 None
9155 } else {
9156 let candidate = PrMessage {
9157 title: first.to_owned(),
9158 body: String::new(),
9159 };
9160 self.judge_pr_language(&winner.worktree, &candidate).await
9161 };
9162 match gt::vet_title(
9163 &text,
9164 decision,
9165 self.state.config.graph.github_english(),
9166 ) {
9167 Ok(title) => {
9168 g.resolved = true;
9169 g.chosen_title = Some(title);
9170 g.choice = Some(GithubTextChoice::MyTitle);
9171 }
9172 Err(cats) if g.asks < 2 => {
9173 ask_again = Some(Some(
9174 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9175 ))
9176 }
9177 Err(_) => {
9178 g.resolved = true;
9179 g.chosen_title = None;
9180 g.choice = Some(GithubTextChoice::Fallback);
9181 }
9182 }
9183 }
9184 },
9185 }
9186 if let Some(rejected) = ask_again {
9187 let w = gt::Withheld {
9190 title: g.title,
9191 body: g.body,
9192 categories: g.categories.clone(),
9193 };
9194 let artifact = match crate::run::write_artifact(
9197 &self.state,
9198 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9199 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9200 ) {
9201 Ok(p) => Some(p),
9202 Err(e) => {
9203 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9204 None
9205 }
9206 };
9207 let (detail, comparison) = gt::question_content(
9208 &w,
9209 &pr.title,
9210 &pr.body,
9211 &self.state.id,
9212 rejected.as_deref(),
9213 artifact.as_deref(),
9214 g.rewrite.as_ref(),
9215 );
9216 let mut q = ask::Question::new(
9217 self.state.id.clone(),
9218 gt::ASK_NODE.to_owned(),
9219 gt::ASK_SEAT.to_owned(),
9220 gt::question_summary(&self.state.config.graph.language, &w),
9221 detail,
9222 gt::question_choices(&w, g.rewrite.as_ref()),
9223 );
9224 q.posting_comparison = Some(comparison);
9225 q.answer_timeout = timeout;
9226 g.question = Some(q.id.clone());
9229 g.asks += 1;
9230 self.state.github_text = Some(g.clone());
9231 self.state.event(
9232 "github-text",
9233 format!("asking the owner about the withheld text ({})", q.short()),
9234 );
9235 self.state.save()?;
9236 store
9237 .put_executable(&mut q)
9238 .context("file the posting-gate question")?;
9239 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9240 tracing::warn!(
9241 "could not notify about posting-gate question {}: {e:#}",
9242 q.short()
9243 );
9244 }
9245 self.park_on_github_text(&g)?;
9246 return Ok(None);
9247 }
9248 }
9249 let resolved = g.chosen_title.clone();
9250 let original = g.choice == Some(GithubTextChoice::Original);
9251 self.state.event(
9253 "github-text",
9254 if original {
9255 format!(
9256 "posting the original text for: {}",
9257 match (g.title, g.body) {
9258 (true, true) => "title, description",
9259 (true, false) => "title",
9260 _ => "description",
9261 }
9262 )
9263 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9264 "posting the saved rewrite title and description".to_owned()
9265 } else if resolved.is_some() {
9266 "posting the owner's replacement title".to_owned()
9267 } else {
9268 "posting the neutral text for the withheld fields".to_owned()
9269 },
9270 );
9271 self.state.github_text = Some(g.clone());
9272 self.state.save()?;
9273 if g.choice == Some(GithubTextChoice::Rewrite)
9274 && let Some(r) = g.rewrite.as_ref()
9275 {
9276 return Ok(Some(PrMessage {
9277 title: r.title.clone(),
9278 body: r.body.clone(),
9279 }));
9280 }
9281 let w = gt::Withheld {
9282 title: g.title,
9283 body: g.body,
9284 categories: g.categories.clone(),
9285 };
9286 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9287 if let Some(t) = resolved {
9288 title = t;
9289 }
9290 let english = gt::LanguageDecision {
9292 title_english: true,
9293 body_english: true,
9294 body_complete: true,
9295 };
9296 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9297 if original {
9298 if g.title {
9301 title = pr.title.clone();
9302 }
9303 if g.body {
9304 body = pr.body.clone();
9305 }
9306 }
9307 Ok(Some(PrMessage { title, body }))
9308 }
9309
9310 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9311 self.state.github_text = Some(g.clone());
9312 self.state.end_pass_parked(ParkCause::Question);
9313 self.state.parked = true;
9314 self.state.event(
9315 "github-text",
9316 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9317 );
9318 self.state.save()
9319 }
9320
9321 async fn judge_pr_language(
9324 &mut self,
9325 cwd: &Path,
9326 pr: &PrMessage,
9327 ) -> Option<crate::github_text::LanguageDecision> {
9328 self.state.config.roles.language_judge.as_ref()?;
9329 if !self.state.config.graph.language_gate() {
9330 return None;
9331 }
9332 let decision =
9333 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9334 self.state.event(
9335 "github-text",
9336 if decision.is_some() {
9337 "language decided by the language judge"
9338 } else {
9339 "language judge unavailable; using built-in heuristics"
9340 },
9341 );
9342 decision
9343 }
9344
9345 async fn merge(&mut self) -> Result<()> {
9346 if self.resource_halt()? || self.hold_base_sync().await? {
9347 return Ok(());
9348 }
9349 if self
9364 .state
9365 .base_sync
9366 .as_ref()
9367 .is_some_and(|s| s.conflict.is_some())
9368 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9369 != Some(RunStatus::Gating)
9370 || !self.state.gate_allows_progress()
9378 {
9379 return Ok(());
9380 }
9381 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9390 return Ok(());
9391 }
9392 let Some(winner) = self.state.winner().cloned() else {
9393 return Ok(());
9394 };
9395 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9396 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9397 self.state.status = RunStatus::Blocked;
9398 self.state.event(
9399 "gate",
9400 "winner changed after gate; fresh verification required",
9401 );
9402 self.state.save()?;
9403 return Ok(());
9404 }
9405 let wait = self.repository_wait();
9406 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9407 return Ok(());
9408 }
9409 let repo = self.state.repo.clone();
9410 let base = self.state.base_branch.clone();
9411 let mode = self.state.config.merge.mode;
9412 let style = self.state.config.merge.style;
9413 let facts = if is_review_run(&self.state) {
9414 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9415 let start = review_base(
9416 &repo,
9417 &self.state.config.merge.remote,
9418 &base,
9419 &self.state.base_commit,
9420 &winner.branch,
9421 )
9422 .await;
9423 branch_facts(&repo, &start, &winner.branch).await
9424 } else {
9425 None
9426 };
9427 let leaked = if is_review_run(&self.state) {
9430 leaked_subjects(&self.state, &winner.branch).await
9431 } else {
9432 Some(Vec::new())
9433 };
9434 let Some(pr) = self
9438 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9439 .await?
9440 else {
9441 return Ok(());
9442 };
9443 let message = pr.commit_message();
9444
9445 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9446 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9447 return Ok(());
9448 }
9449 let outcome = match mode {
9450 MergeMode::None => MergeOutcome {
9451 mode,
9452 ok: true,
9453 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9454 empty: false,
9455 },
9456 MergeMode::Pr | MergeMode::Local
9457 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9458 {
9459 MergeOutcome {
9460 mode,
9461 ok: false,
9462 detail: empty_candidate_detail(&self.state, &base),
9463 empty: true,
9464 }
9465 }
9466 MergeMode::Local => {
9467 let on = git::current_branch(&repo).await?;
9468 if on.as_deref() != Some(base.as_str()) {
9469 MergeOutcome {
9470 mode,
9471 ok: false,
9472 detail: format!(
9473 "{} has {} checked out, not the base branch {base}",
9474 repo.display(),
9475 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9476 ),
9477 empty: false,
9478 }
9479 } else if !git::is_clean(&repo).await? {
9480 MergeOutcome {
9481 mode,
9482 ok: false,
9483 detail: format!("{} is dirty; refusing to merge", repo.display()),
9484 empty: false,
9485 }
9486 } else {
9487 let out = match style {
9488 MergeStyle::Merge => {
9489 git::merge_no_ff(&repo, &winner.branch, &message).await?
9490 }
9491 MergeStyle::Squash => {
9492 git::merge_squash(&repo, &winner.branch, &message).await?
9493 }
9494 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9495 };
9496 MergeOutcome {
9497 mode,
9498 ok: out.ok(),
9499 detail: if out.ok() { out.stdout } else { out.stderr },
9500 empty: false,
9501 }
9502 }
9503 }
9504 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9505 self.state.pr_publication_pending = true;
9506 self.state.save()?;
9507 let out =
9508 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9509 self.state.pr_publication_pending = out.is_err();
9510 MergeOutcome {
9511 mode,
9512 ok: out.is_ok(),
9513 detail: match out {
9514 Ok(url) => url,
9515 Err(e) => format!("{e:#}"),
9516 },
9517 empty: false,
9518 }
9519 }
9520 MergeMode::Pr => {
9521 let remote = self.state.config.merge.remote.clone();
9522 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9523 let pushed = if self.state.foreign_checkout.is_some() {
9524 let refspec = format!("{delivery_head}:refs/heads/{public}");
9525 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9526 } else {
9527 git::push(&winner.worktree, &remote, &winner.branch).await?
9528 };
9529 if !pushed.ok() {
9530 MergeOutcome {
9531 mode,
9532 ok: false,
9533 detail: pushed.stderr,
9534 empty: false,
9535 }
9536 } else {
9537 if self.state.foreign_checkout.is_some() {
9538 self.state.pr_pushed_head = Some(delivery_head.clone());
9539 }
9540 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9546 let out = match pr_merge_plan(found) {
9547 PrPlan::Create => {
9548 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9549 .await
9550 }
9551 PrPlan::Adopt { url, title } => {
9552 self.state
9553 .event("merge", format!("Pr: adopted open pull request {url}"));
9554 if self.state.github_text.as_ref().is_some_and(|g| {
9555 g.resolved
9556 && g.choice == Some(GithubTextChoice::Rewrite)
9557 && g.rewrite.is_some()
9558 }) {
9559 land::set_pr_text_verbatim(
9562 &winner.worktree,
9563 &url,
9564 &pr.title,
9565 &pr.body,
9566 )
9567 .await
9568 .map(|()| url)
9569 } else {
9570 if title != pr.title
9571 && (!is_review_run(&self.state)
9572 || leaked
9573 .as_deref()
9574 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9575 && let Err(e) =
9576 if self.state.github_text.as_ref().is_some_and(|g| {
9577 g.resolved
9578 && g.title
9579 && g.choice == Some(GithubTextChoice::Original)
9580 }) {
9581 land::set_pr_title_verbatim(
9583 &winner.worktree,
9584 &url,
9585 &pr.title,
9586 )
9587 .await
9588 } else {
9589 land::set_pr_title(
9590 &mut self.state,
9591 &winner.worktree,
9592 &url,
9593 &pr.title,
9594 )
9595 .await
9596 }
9597 {
9598 tracing::warn!("could not refresh title of {url}: {e:#}");
9599 self.state
9600 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9601 }
9602 Ok(url)
9603 }
9604 }
9605 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9606 };
9607 match out {
9608 Ok(url) => MergeOutcome {
9609 mode,
9610 ok: true,
9611 detail: url,
9612 empty: false,
9613 },
9614 Err(e) => MergeOutcome {
9615 mode,
9616 ok: false,
9617 detail: e.to_string(),
9618 empty: false,
9619 },
9620 }
9621 }
9622 }
9623 };
9624
9625 self.state.status = match (mode, outcome.ok) {
9626 (MergeMode::None, _) => RunStatus::Ready,
9627 (_, true) if outcome.empty => RunStatus::Ready,
9628 (MergeMode::Pr, true) => RunStatus::Ready,
9629 (_, true) => RunStatus::Merged,
9630 (_, false) => RunStatus::Blocked,
9631 };
9632 self.state.event(
9633 "merge",
9634 format!(
9635 "{:?}: {}",
9636 mode,
9637 outcome.detail.lines().next().unwrap_or("")
9638 ),
9639 );
9640 self.state.merge = Some(outcome);
9641 self.state.save()?;
9642
9643 if self.state.config.graph.land
9649 && mode == MergeMode::Pr
9650 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9651 {
9652 self.run_land().await?;
9653 }
9654 self.settle_questions();
9659 Ok(())
9660 }
9661
9662 async fn run_land(&mut self) -> Result<()> {
9673 crate::pr_head::ensure_owner(&self.state)?;
9674 let url = self
9675 .state
9676 .pr
9677 .as_ref()
9678 .map(|pr| pr.url.clone())
9679 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9680 .unwrap_or_default();
9681 let url = url.lines().next().unwrap_or("").trim().to_owned();
9682 if !url.starts_with("http") {
9683 return Ok(());
9684 }
9685 loop {
9688 match land::land(&mut self.state, &url).await {
9689 Ok(_)
9690 if self
9691 .state
9692 .merge_repairs
9693 .last()
9694 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9695 {
9696 if self.repair_merge_review().await? {
9697 continue;
9698 }
9699 }
9700 Ok(_) if self.state.resource_block.is_some() => {
9701 return self.stop_for_resource();
9702 }
9703 Ok(pr) if self.state.parked => {
9704 let _ = pr;
9708 }
9709 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9710 self.state.status = RunStatus::Superseded;
9711 self.state.save()?;
9712 }
9713 Ok(pr) => {
9714 self.state.status = match pr.state {
9715 land::PrLifecycle::Merged => RunStatus::Merged,
9716 _ => RunStatus::Blocked,
9717 };
9718 if bump::should_release_bump(self.state.status)
9725 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9726 {
9727 self.state
9733 .event("bump", format!("release bump skipped: {e:#}"));
9734 }
9735 if self.state.status == RunStatus::Merged {
9738 crate::followup::after_merge(&mut self.state, &pr.url).await;
9739 }
9740 self.state.save()?;
9741 }
9742 Err(e) => {
9743 if e.is::<crate::repo_lock::ResourceBlocked>() {
9744 return Err(e);
9745 }
9746 self.state.status = RunStatus::Blocked;
9747 self.state.event("land", format!("gave up: {e}"));
9748 self.state.save()?;
9749 }
9750 }
9751 break;
9752 }
9753 Ok(())
9754 }
9755
9756 async fn repair_merge_review(&mut self) -> Result<bool> {
9759 use crate::run::MergeRepairPhase as Phase;
9760 crate::pr_head::ensure_owner(&self.state)?;
9761 let i = self.state.merge_repairs.len() - 1;
9762 let request = self.state.merge_repairs[i].clone();
9763 if request.phase == Phase::Fixing {
9764 let restored = match self.state.winner().cloned() {
9765 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9766 None => Err(anyhow::anyhow!("repair has no winning branch")),
9767 };
9768 let why = match restored {
9769 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9770 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9771 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9772 };
9773 self.hold_merge_repair(i, why)?;
9774 return Ok(false);
9775 }
9776 if request.phase == Phase::Reviewing && self.state.gate_ran {
9781 let winner = self
9782 .state
9783 .winner()
9784 .context("repair has no winning branch")?;
9785 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9786 if self.state.gate_head.as_deref() != Some(head.as_str())
9787 || !git::is_clean(&winner.worktree).await?
9788 {
9789 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9790 return Ok(false);
9791 }
9792 }
9793 let failure = if i >= self.state.config.graph.land_rounds
9794 || self.state.config.graph.review_rounds == 0
9795 {
9796 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9797 } else {
9798 None
9799 };
9800 if let Some(why) = failure {
9801 self.hold_merge_repair(i, why)?;
9802 return Ok(false);
9803 }
9804 if request.phase == Phase::Requested {
9805 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9806 Ok(pr) => pr,
9807 Err(e) => {
9808 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9809 return Ok(false);
9810 }
9811 };
9812 self.state.merge_repairs[i].phase = Phase::Fixing;
9813 self.state.land_approval = None;
9814 self.state.land_armed_head = None;
9815 self.state.save()?;
9816 let evidence = serde_json::to_string_pretty(&request)?;
9819 let reason = format!(
9820 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9821 );
9822 let budget = self.state.config.graph.land_rounds;
9823 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9824 match result {
9825 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9826 Ok(land::Fixed::Deferred) => {
9827 let worktree = self
9831 .state
9832 .winner()
9833 .context("repair has no winning branch")?
9834 .worktree
9835 .clone();
9836 self.graft_restore(&worktree, &request.head).await?;
9837 self.state.merge_repairs[i].phase = Phase::Requested;
9838 self.stop_for_resource()?;
9839 return Ok(false);
9840 }
9841 Ok(land::Fixed::Failed(why)) => {
9842 self.hold_merge_repair(
9843 i,
9844 format!(
9845 "repair failed: {why}; inspect the branch and run an explicit review"
9846 ),
9847 )?;
9848 return Ok(false);
9849 }
9850 Err(e) => {
9851 self.hold_merge_repair(
9852 i,
9853 format!(
9854 "repair failed: {e:#}; inspect the branch and run an explicit review"
9855 ),
9856 )?;
9857 return Ok(false);
9858 }
9859 }
9860 self.begin_merge_repair_review(i)?;
9864 }
9865 self.review_loop().await?;
9866 self.gate().await?;
9867 if self.resource_halt()? || self.state.parked {
9868 return Ok(false);
9869 }
9870 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9871 != Some(RunStatus::Gating)
9872 || !self.state.gate_allows_progress()
9873 {
9874 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9875 return Ok(false);
9876 }
9877 let winner = self
9880 .state
9881 .winner()
9882 .cloned()
9883 .context("repair has no winning branch")?;
9884 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9885 if self.state.gate_head.as_deref() != Some(after.as_str())
9886 || !git::is_clean(&winner.worktree).await?
9887 {
9888 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9889 return Ok(false);
9890 }
9891 let expected = self
9892 .state
9893 .pr_pushed_head
9894 .as_deref()
9895 .unwrap_or(&request.head)
9896 .to_owned();
9897 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9898 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9901 return Ok(false);
9902 }
9903 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9904 let refspec = format!("{after}:refs/heads/{public}");
9905 let push = git::git_raw(
9906 &winner.worktree,
9907 &["push", &lease, &self.state.config.merge.remote, &refspec],
9908 )
9909 .await;
9910 match push {
9911 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9912 Ok(out) => {
9913 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9914 return Ok(false);
9915 }
9916 Err(e) => {
9917 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9918 return Ok(false);
9919 }
9920 }
9921 self.state.merge_repairs[i].phase = Phase::Complete;
9922 self.state.status = RunStatus::Landing;
9923 self.state.event(
9924 "repair",
9925 "new review and gate completed; fresh owner merge decision required",
9926 );
9927 self.state.save()?;
9928 Ok(true)
9929 }
9930
9931 fn merge_repair_artifacts(&self) -> PathBuf {
9932 let artifacts = agent::artifacts_dir(&self.state.dir());
9933 match self
9934 .state
9935 .merge_repairs
9936 .last()
9937 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9938 {
9939 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9940 None => artifacts,
9941 }
9942 }
9943
9944 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9945 if let Some(e2e) = self.state.head_e2e.take() {
9946 self.state
9947 .verification_history
9948 .push(crate::run::VerificationEvidence {
9949 superseded_by: None,
9950 archived_at: Some(Timestamp::now()),
9951 fixes_spent: Some(self.state.gate_fixes.len()),
9952 gate_head: None,
9953 gate_ran: false,
9954 gate: Vec::new(),
9955 e2e: Some(e2e),
9956 });
9957 }
9958 self.state.reviews.clear();
9959 self.state.gate.clear();
9960 self.state.gate_ran = false;
9961 self.state.gate_head = None;
9962 self.state.gate_fixes.clear();
9963 self.state.gate_repair_e2e = None;
9964 self.state.pre_gate.clear();
9965 self.state.pre_gate_commit = None;
9966 self.state.contested_handoff = None;
9967 self.state.land_reruns = None;
9968 self.state.land_approval = None;
9969 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9970 self.state.status = RunStatus::Reviewing;
9971 self.state.save()
9972 }
9973
9974 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9975 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9976 self.state.merge_repairs[index].failure = Some(why.clone());
9977 self.state.status = RunStatus::Blocked;
9978 self.state.event("repair", why);
9979 self.state.save()
9980 }
9981
9982 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9986 if let Some(existing) = self.state.seats.get(key)
9987 && existing.agent == agent
9988 {
9989 return existing.clone();
9990 }
9991 let fresh = if self.state.seats.contains_key(key) {
9994 handover_seat(key, agent, self.state.next_seat_seed())
9995 } else {
9996 SeatState::new(key, agent, self.state.seed)
9997 };
9998 self.state.seats.insert(key.to_owned(), fresh.clone());
9999 fresh
10000 }
10001
10002 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10008 match self.state.seats.get(key) {
10009 Some(s) if s.agent != spec.id => {
10010 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10011 }
10012 _ => spec,
10013 }
10014 }
10015
10016 fn view(&self, c: &Candidate) -> CandidateView {
10018 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10019 .unwrap_or_default();
10020 let (patch, _) = blind::sanitize_patch(
10021 &format!("candidate {} patch", c.label),
10022 &raw,
10023 &self.state.config.blind,
10024 );
10025 CandidateView {
10026 label: c.label,
10027 branch: c.branch.clone(),
10028 summary: c.summary.clone(),
10029 stat: c.stat.clone(),
10030 patch,
10031 }
10032 }
10033
10034 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10036 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10037 prompt::judge(
10038 "(see above)",
10039 &views,
10040 self.roles.judges.len(),
10041 base_short,
10042 "en",
10043 )
10044 }
10045
10046 fn vote_prompt_full(
10052 &self,
10053 j: usize,
10054 labels: &[char],
10055 language: &str,
10056 candidates: &[Candidate],
10057 base_short: &str,
10058 ) -> String {
10059 let mut text = format!(
10060 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10061 prompt::final_vote(labels, language),
10062 self.state.instruction,
10063 self.candidate_block(candidates, base_short)
10064 );
10065 if let Some(own) = self
10066 .state
10067 .judgements
10068 .get(j)
10069 .filter(|r| !r.ranking.is_empty())
10070 {
10071 let reasons = own
10072 .reasons
10073 .iter()
10074 .map(|(k, v)| format!("- {k}: {v}"))
10075 .collect::<Vec<_>>()
10076 .join("\n");
10077 text.push_str(&format!(
10078 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10079 own.ranking.iter().collect::<String>(),
10080 if reasons.is_empty() {
10081 ""
10082 } else {
10083 ", because:\n"
10084 }
10085 ));
10086 }
10087 if !self.state.deliberation.is_empty() {
10088 text.push_str("\n# What was argued before this vote\n");
10089 for t in self.transcript(&[], j) {
10090 text.push_str(&format!(
10091 "\n## {}{}\n\n{}\n",
10092 t.who,
10093 if t.is_self { " (you)" } else { "" },
10094 t.body.trim()
10095 ));
10096 }
10097 }
10098 text
10099 }
10100
10101 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10108 let mut turns = Vec::new();
10109 for j in &self.state.judgements {
10110 if j.ranking.is_empty() {
10111 continue;
10112 }
10113 let reasons = j
10114 .reasons
10115 .iter()
10116 .map(|(k, v)| format!("- {k}: {v}"))
10117 .collect::<Vec<_>>()
10118 .join("\n");
10119 turns.push(Turn {
10120 who: format!("Judge {} (opening ranking)", j.judge),
10121 is_self: j.judge == self_idx + 1,
10122 body: format!(
10123 "Ranked {}{}{reasons}",
10124 j.ranking.iter().collect::<String>(),
10125 if reasons.is_empty() {
10126 ""
10127 } else {
10128 ", because:\n"
10129 }
10130 ),
10131 });
10132 }
10133 for t in self
10134 .state
10135 .deliberation
10136 .iter()
10137 .flat_map(|r| r.turns.iter())
10138 .chain(current)
10139 {
10140 turns.push(Turn {
10141 who: format!("Judge {}", t.judge),
10142 is_self: t.judge == self_idx + 1,
10143 body: t.body.clone(),
10144 });
10145 }
10146 turns
10147 }
10148}
10149
10150fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10152 agent::has_session(spec.kind, seat, sessions)
10153}
10154
10155fn next_untried_in_roster<'a>(
10176 roster: &'a [AgentSpec],
10177 start: usize,
10178 tried: &BTreeSet<String>,
10179) -> Option<&'a AgentSpec> {
10180 roster
10181 .get(start + 1..)?
10182 .iter()
10183 .find(|s| !tried.contains(&s.id))
10184}
10185
10186fn pick_successor<'a>(
10207 roster: &'a [AgentSpec],
10208 start: usize,
10209 tried: &BTreeSet<String>,
10210 carried: Option<&BTreeSet<String>>,
10211 others: &BTreeSet<String>,
10212) -> Option<&'a AgentSpec> {
10213 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10214 roster
10215 .get(start + 1..)
10216 .and_then(|tail| {
10217 tail.iter()
10218 .filter(free)
10219 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10220 })
10221 .or_else(|| {
10222 carried
10223 .is_some()
10224 .then(|| roster.iter().find(free))
10225 .flatten()
10226 })
10227 .or_else(|| match carried {
10228 Some(c) => next_for_seat(roster, start, tried, c),
10229 None => next_untried_in_roster(roster, start, tried),
10230 })
10231}
10232
10233fn next_for_seat<'a>(
10243 roster: &'a [AgentSpec],
10244 start: usize,
10245 round_tried: &BTreeSet<String>,
10246 carried_failed: &BTreeSet<String>,
10247) -> Option<&'a AgentSpec> {
10248 roster
10249 .get(start + 1..)?
10250 .iter()
10251 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10252 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10253}
10254
10255fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10261 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10262 return spec;
10263 };
10264 let ok = |id: &str| !h.failed.contains(id);
10265 if let Some(last) = h.last_ok.as_deref()
10266 && ok(last)
10267 && let Some(s) = roster.iter().find(|s| s.id == last)
10268 {
10269 return s.clone();
10270 }
10271 if ok(&spec.id) {
10272 return spec;
10273 }
10274 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10275 roster
10276 .get(from + 1..)
10277 .into_iter()
10278 .flatten()
10279 .chain(roster.iter())
10280 .find(|s| ok(&s.id))
10281 .cloned()
10282 .unwrap_or(spec)
10283}
10284
10285pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10289 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10290}
10291
10292const TIMED_OUT: &str = "timed out";
10295
10296impl FailClass {
10297 fn of(out: &AgentOutcome) -> Option<Self> {
10299 match out {
10300 AgentOutcome::Ok(_) => None,
10301 AgentOutcome::Quota(_) => Some(Self::Quota),
10302 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10303 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10304 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10305 AgentOutcome::ResourceBlocked(_) => None,
10308 }
10309 }
10310
10311 fn stem_word(&self) -> &'static str {
10313 match self {
10314 Self::Quota => "quota",
10315 _ => "handover",
10316 }
10317 }
10318}
10319
10320fn failure_signature(msg: &str) -> String {
10324 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10325 if let Some(code) = line
10328 .strip_prefix("process exited with no usable output (exit code ")
10329 .and_then(|s| s.strip_suffix(')'))
10330 .and_then(|s| s.parse::<i32>().ok())
10331 {
10332 line = format!("exited with some({code}) and no usable output");
10333 } else if matches!(
10334 line.as_str(),
10335 "process exited with no usable output (no exit code reported)"
10336 | "process exited with no usable output (exit code unavailable)"
10337 ) {
10338 line = "exited with none and no usable output".to_owned();
10339 }
10340 let mut out = Vec::new();
10341 for word in line.split_whitespace() {
10342 if word.contains('/') || word.contains('\\') {
10343 out.push("<path>".to_owned());
10344 continue;
10345 }
10346 let mut w = String::new();
10347 let mut in_digits = false;
10348 for c in word.chars() {
10349 if c.is_ascii_digit() {
10350 if !in_digits {
10351 w.push('#');
10352 }
10353 in_digits = true;
10354 } else {
10355 in_digits = false;
10356 w.push(c);
10357 }
10358 }
10359 out.push(w);
10360 }
10361 out.join(" ").chars().take(120).collect()
10362}
10363
10364fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10371 match cur {
10372 FailClass::Quota | FailClass::Timeout => true,
10373 FailClass::Other(_) => prev != Some(cur),
10374 }
10375}
10376
10377fn fail_reason(out: &AgentOutcome) -> String {
10379 match out {
10380 AgentOutcome::Ok(_) => String::new(),
10381 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10382 AgentOutcome::Dropped(o) => format!(
10383 "the CLI dropped the stream ({})",
10384 o.dropped
10385 .as_ref()
10386 .map(|d| d.why.as_str())
10387 .unwrap_or("it ended without delivering its answer")
10388 ),
10389 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10390 e.lines().next().unwrap_or("").chars().take(160).collect()
10391 }
10392 }
10393}
10394
10395pub(crate) fn record_handover(
10398 state: &mut RunState,
10399 node: &str,
10400 seat: &str,
10401 from: &str,
10402 to: &str,
10403 class: &FailClass,
10404 reason: &str,
10405) {
10406 let message = if *class == FailClass::Quota {
10407 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10408 } else {
10409 format!("{seat}: handed over {from} -> {to} ({reason})")
10410 };
10411 state.event(node, message);
10412 state.handovers.push(Handover {
10413 at: Timestamp::now(),
10414 node: node.to_owned(),
10415 seat: seat.to_owned(),
10416 from: from.to_owned(),
10417 to: to.to_owned(),
10418 reason: reason.to_owned(),
10419 });
10420}
10421
10422fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10436 commands.iter().any(|c| c.exit_code.is_none())
10437}
10438
10439fn verified_noop_claim(
10452 usable: bool,
10453 commands: &[agent::CommandEvidence],
10454 text: &str,
10455) -> Option<String> {
10456 (usable && !has_unconfirmed_command(commands))
10457 .then(|| verdict::verified_noop(text))
10458 .flatten()
10459}
10460
10461fn short(commit: &str) -> String {
10462 commit.chars().take(7).collect()
10463}
10464
10465fn make_executable(path: &Path) -> Result<()> {
10466 #[cfg(unix)]
10467 {
10468 use std::os::unix::fs::PermissionsExt as _;
10469 let mut perms = std::fs::metadata(path)?.permissions();
10470 perms.set_mode(0o755);
10471 std::fs::set_permissions(path, perms)?;
10472 }
10473 #[cfg(not(unix))]
10474 {
10475 let _ = path;
10476 }
10477 Ok(())
10478}
10479
10480struct WaveCtx<'a> {
10487 run: &'a str,
10490 node: &'a str,
10492 prompts: &'a Prompts,
10493 cache: Option<&'a Path>,
10495 round: Option<usize>,
10498 carry_seats: bool,
10502 pauses: Option<(&'a Pause, &'a Pause)>,
10505}
10506
10507fn repository_wait(
10509 state: &RunState,
10510 pauses: Option<(&Pause, &Pause)>,
10511) -> crate::repo_lock::WaitPolicy {
10512 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10513 let inherited = crate::repo_lock::WaitPolicy::current();
10514 crate::repo_lock::WaitPolicy::new(
10515 Duration::from_secs(state.config.graph.timeout_resource_wait),
10516 move || {
10517 inherited.cancelled()
10518 || pauses
10519 .as_ref()
10520 .is_some_and(|(a, b)| a.parked() || b.parked())
10521 },
10522 )
10523}
10524
10525fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10526 if error.is::<crate::repo_lock::ResourceBlocked>() {
10527 state.resource_block = Some(crate::run::ResourceBlock {
10528 cause: crate::run::ResourceBlockCause::Other,
10529 node: "repository".into(),
10530 kind: if error.to_string().contains("disk pressure") {
10531 "disk pressure"
10532 } else {
10533 "repository coordination"
10534 }
10535 .into(),
10536 detail: error.to_string(),
10537 at: Timestamp::now(),
10538 });
10539 state.event("resource", error.to_string());
10540 }
10541}
10542
10543async fn coordinate_operation(
10544 state: &mut RunState,
10545 cwd: &Path,
10546 wait: &crate::repo_lock::WaitPolicy,
10547) -> Result<Option<crate::repo_lock::SeatLease>> {
10548 let result =
10549 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10550 .await;
10551 if let Err(e) = &result {
10552 record_repository_block(state, e);
10553 }
10554 result
10555}
10556
10557async fn run_one(
10559 job: SeatJob,
10560 sem: Arc<Semaphore>,
10561 ctx: &WaveCtx<'_>,
10562 state: &mut RunState,
10563 attempt: usize,
10564) -> (SeatState, AgentOutcome) {
10565 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10566 .await
10567 .pop()
10568 .expect("one job in, one result out");
10569 (seat, out)
10570}
10571
10572async fn wave(
10578 jobs: Vec<SeatJob>,
10579 sem: Arc<Semaphore>,
10580 ctx: &WaveCtx<'_>,
10581 state: &mut RunState,
10582 attempt: usize,
10583) -> Vec<(usize, SeatState, AgentOutcome)> {
10584 let WaveCtx {
10585 run,
10586 node,
10587 prompts,
10588 cache,
10589 round,
10590 carry_seats: _,
10591 pauses,
10592 } = *ctx;
10593 for job in &jobs {
10594 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10595 }
10596 if let Err(e) = state.save() {
10597 tracing::warn!("could not persist in-progress seats: {e:#}");
10602 }
10603 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10619 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10623 let cache_guard = if let Some(cache_dir) = cache {
10624 if jobs_had_a_writer {
10625 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10626 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10627 match acquire_cache_lease(
10628 state,
10629 cache_dir,
10630 &owner,
10631 resource_limit,
10632 node,
10633 node,
10634 &cancel,
10635 )
10636 .await
10637 {
10638 Ok(Some(g)) => Some(g),
10639 Ok(None) => {
10642 if let Some(block) = &mut state.resource_block {
10643 block.cause = crate::run::ResourceBlockCause::Cancelled {
10644 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10645 };
10646 }
10647 return block_wave(
10648 jobs,
10649 state,
10650 node,
10651 attempt,
10652 "parked while waiting for the shared build cache",
10653 );
10654 }
10655 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10658 Err(e) => {
10659 let why = format!("{e:#}");
10663 state.event(
10664 "resource",
10665 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10666 );
10667 return block_wave(jobs, state, node, attempt, &why);
10668 }
10669 }
10670 } else {
10671 None
10672 }
10673 } else {
10674 None
10675 };
10676 state.seats_spawning(node, attempt);
10679 if let Err(e) = state.save() {
10680 tracing::warn!("could not persist the start of a wave: {e:#}");
10681 }
10682 let slot_limit = resource_limit
10686 + jobs
10687 .iter()
10688 .map(|j| j.timeout)
10689 .max()
10690 .unwrap_or_default()
10691 .saturating_mul(jobs.len() as u32);
10692 let mut set = tokio::task::JoinSet::new();
10693 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10696 let overlay = prompts.overlay(node);
10697 let repository_wait = repository_wait(state, pauses);
10698 for (i, mut job) in jobs.into_iter().enumerate() {
10699 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10700 if cache.is_some() {
10701 job.prompt.push('\n');
10702 job.prompt
10703 .push_str(&prompt::build_cache_note(node, job.allow_write));
10704 }
10705 let sem = Arc::clone(&sem);
10706 let run = run.to_owned();
10707 let node = node.to_owned();
10708 let attachments = if node == "implement" {
10711 state.attachments.clone()
10712 } else {
10713 Vec::new()
10714 };
10715 let cache = cache
10726 .filter(|_| job.allow_write && cache_guard.is_some())
10727 .map(Path::to_path_buf);
10728 let spawned_tx = spawned_tx.clone();
10729 let repository_wait = repository_wait.clone();
10730 let coordination = state.config.coordination.clone();
10731 let fetched_base = state.fetched_base.clone();
10732 set.spawn(async move {
10733 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10737 Ok(p) => p,
10738 Err(_) => {
10739 return (
10740 i,
10741 job.seat,
10742 AgentOutcome::ResourceBlocked(format!(
10743 "no parallel slot became free within {}s",
10744 slot_limit.as_secs()
10745 )),
10746 false,
10747 );
10748 }
10749 };
10750 spawned_tx.send(job.seat.key.clone()).ok();
10751 let mut seat = job.seat;
10752 let out = repository_wait
10753 .scope(agent::invoke(
10754 &job.spec,
10755 &mut seat,
10756 &Invocation {
10757 coordination: coordination.as_ref(),
10758 fetched_base: fetched_base.as_ref(),
10759 cwd: &job.cwd,
10760 prompt: &job.prompt,
10761 timeout: job.timeout,
10762 allow_write: job.allow_write,
10763 sessions: job.sessions,
10764 artifacts: &job.artifacts,
10765 stem: &job.stem,
10766 run: &run,
10767 node: &node,
10768 cache_dir: cache.as_deref(),
10769 attachments: &attachments,
10770 writable: &[],
10771 },
10772 ))
10773 .await;
10774 let interrupted = out
10775 .as_ref()
10776 .err()
10777 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10778 let out = match out {
10779 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10780 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10781 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10789 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10790 Ok(o) => AgentOutcome::Failed(format!(
10791 "process exited with no usable output ({})",
10792 crate::notices::exit_status(o.exit_code)
10793 )),
10794 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10795 AgentOutcome::ResourceBlocked(e.to_string())
10796 }
10797 Err(e) => AgentOutcome::Failed(e.to_string()),
10798 };
10799 (i, seat, out, interrupted)
10800 });
10801 }
10802 drop(spawned_tx);
10803 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10804 loop {
10805 let joined = tokio::select! {
10806 Some(key) = spawned_rx.recv() => {
10807 state.seat_spawned(&key);
10809 if let Err(e) = state.save() {
10810 tracing::warn!("could not persist a seat's start: {e:#}");
10811 }
10812 continue;
10813 }
10814 joined = set.join_next() => joined,
10815 };
10816 let Some(joined) = joined else {
10817 break;
10818 };
10819 let (i, seat, out, interrupted) = match joined {
10820 Ok(v) => v,
10821 Err(e) => {
10825 tracing::error!("agent task panicked: {e}");
10826 continue;
10827 }
10828 };
10829 state.seat_finished(&seat.key);
10830 if interrupted
10831 && node == "implement"
10832 && let Some(label) = state
10833 .candidates
10834 .iter()
10835 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10836 .map(|candidate| candidate.label)
10837 {
10838 state.candidate_preservation.insert(
10839 label,
10840 "interrupted implementation awaiting persistence".into(),
10841 );
10842 state.interrupted_implementations.insert(label);
10843 }
10844 if let AgentOutcome::ResourceBlocked(why) = &out
10845 && state.resource_block.is_none()
10846 {
10847 state.resource_block = Some(crate::run::ResourceBlock {
10848 cause: crate::run::ResourceBlockCause::Other,
10849 node: node.to_owned(),
10850 kind: if why.contains("disk pressure") {
10851 "disk pressure"
10852 } else if why.contains("repository coordination") {
10853 "repository coordination"
10854 } else {
10855 "parallel slot"
10856 }
10857 .to_owned(),
10858 detail: why.clone(),
10859 at: Timestamp::now(),
10860 });
10861 let activity = if interrupted {
10862 "agent execution interrupted"
10863 } else {
10864 "no agent was started"
10865 };
10866 state.event("resource", format!("{node}: {activity} - {why}"));
10867 }
10868 record_jobs(state, node, round, &seat.key, &out);
10869 if let Err(e) = state.save() {
10870 tracing::warn!("could not persist a seat's completion: {e:#}");
10871 }
10872 if collected.len() <= i {
10873 collected.resize_with(i + 1, || None);
10874 }
10875 collected[i] = Some((i, seat, out));
10876 }
10877 if state
10883 .active
10884 .values()
10885 .any(|a| a.node == node && a.attempt == attempt)
10886 {
10887 state
10888 .active
10889 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10890 if let Err(e) = state.save() {
10891 tracing::warn!("could not persist the end of a wave: {e:#}");
10892 }
10893 }
10894 if let Some(cache_dir) = cache
10901 && jobs_had_a_writer
10902 {
10903 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10904 }
10905 if let Some(guard) = cache_guard {
10906 guard.release();
10907 }
10908 collected.into_iter().flatten().collect()
10909}
10910
10911fn record_jobs(
10922 state: &mut RunState,
10923 node: &str,
10924 round: Option<usize>,
10925 seat: &str,
10926 out: &AgentOutcome,
10927) {
10928 let commands: &[agent::CommandEvidence] = match out {
10929 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10930 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10931 };
10932 let checked_at = Timestamp::now();
10933 for c in commands {
10934 state.jobs.push(JobRecord {
10935 node: node.to_owned(),
10936 round,
10937 seat: seat.to_owned(),
10938 id: c.id.clone(),
10939 description: c.description.clone(),
10940 checked_at,
10941 status: match c.exit_code {
10942 Some(0) => JobStatus::Completed,
10943 Some(_) => JobStatus::Failed,
10944 None => JobStatus::Unknown,
10945 },
10946 exit_code: c.exit_code,
10947 result_summary: c.result_summary.clone(),
10948 source: c.source.clone(),
10949 });
10950 }
10951}
10952
10953fn round_is_clean(
10974 blocking: usize,
10975 e2e_ok: bool,
10976 answered: usize,
10977 expected: usize,
10978 quota_missing: usize,
10979 policy: IncompleteReviewPolicy,
10980) -> bool {
10981 if blocking != 0 || !e2e_ok {
10982 return false;
10983 }
10984 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10985 return true;
10986 }
10987 answered > 0 && expected - answered <= quota_missing
10988}
10989
10990pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11014 if max_rounds == 0
11015 || reviews.last().is_some_and(|r| {
11016 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11017 })
11018 {
11019 return Some(RunStatus::Gating);
11020 }
11021 let last = reviews.last()?;
11022 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11023 if reviews.len() < max_rounds && !stagnant {
11024 return None;
11025 }
11026 if last.incomplete() && last.blocking == 0 {
11027 return Some(RunStatus::Blocked);
11028 }
11029 if last.e2e_status() == E2eStatus::ResourceBlocked
11030 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11031 {
11032 return None;
11033 }
11034 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11035 RunStatus::Gating
11036 } else {
11037 RunStatus::Blocked
11038 })
11039}
11040
11041pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11056 if nudged {
11057 (full / 4).max(Duration::from_secs(120)).min(full)
11058 } else {
11059 full
11060 }
11061}
11062
11063#[allow(clippy::too_many_arguments)]
11092async fn ask_json_wave<T>(
11093 jobs: Vec<SeatJob>,
11094 sem: Arc<Semaphore>,
11095 retries: usize,
11096 roster: &[AgentSpec],
11097 ctx: &WaveCtx<'_>,
11098 losses: &mut Vec<QuotaLoss>,
11099 state: &mut RunState,
11100 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11101) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11102where
11103 T: serde::de::DeserializeOwned + Send + 'static,
11104{
11105 ask_wave_with(
11106 jobs,
11107 sem,
11108 retries,
11109 roster,
11110 ctx,
11111 losses,
11112 state,
11113 &|text: &str| {
11114 let v = verdict::extract_json::<T>(text)?;
11115 validate(&v)?;
11116 Ok(v)
11117 },
11118 )
11119 .await
11120}
11121
11122#[allow(clippy::too_many_arguments)]
11131async fn ask_wave_with<T>(
11132 jobs: Vec<SeatJob>,
11133 sem: Arc<Semaphore>,
11134 retries: usize,
11135 roster: &[AgentSpec],
11136 ctx: &WaveCtx<'_>,
11137 losses: &mut Vec<QuotaLoss>,
11138 state: &mut RunState,
11139 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11140) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11141where
11142 T: Send + 'static,
11143{
11144 let n = jobs.len();
11145 let originals: Vec<SeatJob> = jobs;
11146 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11147 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11148 let mut nudges: Vec<usize> = vec![0; n];
11155 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11159 let mut tried: Vec<BTreeSet<String>> = specs
11160 .iter()
11161 .map(|s| BTreeSet::from([s.id.clone()]))
11162 .collect();
11163 let starts: Vec<usize> = specs
11164 .iter()
11165 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11166 .collect();
11167 let carry = ctx.carry_seats && !roster.is_empty();
11168 let carried: Vec<BTreeSet<String>> = originals
11171 .iter()
11172 .map(|j| {
11173 state
11174 .seat_history
11175 .get(&j.seat.key)
11176 .filter(|_| carry)
11177 .map(|h| h.failed.clone())
11178 .unwrap_or_default()
11179 })
11180 .collect();
11181 let mut prev: Vec<Option<FailClass>> = originals
11182 .iter()
11183 .map(|j| {
11184 state
11185 .seat_history
11186 .get(&j.seat.key)
11187 .filter(|_| carry)
11188 .and_then(|h| h.last_fail.clone())
11189 })
11190 .collect();
11191 let next_agent =
11192 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11193 let others: BTreeSet<String> = specs
11194 .iter()
11195 .enumerate()
11196 .filter(|(j, _)| *j != i)
11197 .map(|(_, s)| s.id.clone())
11198 .collect();
11199 pick_successor(
11200 roster,
11201 starts[i],
11202 tried,
11203 carry.then(|| &carried[i]),
11204 &others,
11205 )
11206 .cloned()
11207 };
11208 let mut fresh: Vec<Option<String>> = vec![None; n];
11209 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11210 let mut pending: Vec<usize> = (0..n).collect();
11211
11212 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11217 for round in 0..max_rounds {
11218 if pending.is_empty() {
11219 break;
11220 }
11221 let mut batch = Vec::with_capacity(pending.len());
11222 let mut renudged: Vec<&str> = Vec::new();
11223 for &i in &pending {
11224 let src = &originals[i];
11225 let full: &str = match &src.handover {
11230 Some(h) if specs[i].id != src.spec.id => h,
11231 _ => &src.prompt,
11232 };
11233 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11236 (
11237 full.to_owned(),
11238 src.timeout,
11239 format!("{}-{word}-{}", src.stem, specs[i].id),
11240 )
11241 } else if nudges[i] == 0 {
11242 (full.to_owned(), src.timeout, src.stem.clone())
11243 } else {
11244 renudged.push(src.seat.key.as_str());
11245 let why = done[i]
11246 .as_ref()
11247 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11248 .unwrap_or_else(|| "no parsable answer".to_owned());
11249 let nudge = prompt::nudge(&why);
11250 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11251 let prompt = if nudged {
11252 nudge
11253 } else {
11254 format!("{full}\n\n---\n\n{nudge}")
11255 };
11256 (
11257 prompt,
11258 retry_budget(src.timeout, nudged),
11259 format!("{}-retry{}", src.stem, nudges[i]),
11260 )
11261 };
11262 batch.push(SeatJob {
11263 spec: specs[i].clone(),
11264 seat: seats[i].clone(),
11265 cwd: src.cwd.clone(),
11266 prompt,
11267 timeout,
11268 allow_write: src.allow_write,
11269 sessions: src.sessions,
11270 artifacts: src.artifacts.clone(),
11271 stem,
11272 handover: None,
11273 });
11274 }
11275
11276 if !renudged.is_empty() {
11277 state.event(
11278 ctx.node,
11279 format!("retry {round}: re-asking {}", renudged.join(", ")),
11280 );
11281 }
11282 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11283 let mut still = Vec::new();
11284 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11285 seats[i] = seat;
11286 if let AgentOutcome::ResourceBlocked(why) = &out {
11289 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11290 continue;
11291 }
11292 let class = FailClass::of(&out);
11293 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11296 && nudges[i] < retries
11297 && !roster.is_empty();
11298 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11299 let next = should_hand_over(prev[i].as_ref(), &cur)
11300 .then(|| next_agent(i, &tried[i], &specs))
11301 .flatten();
11302 if carry {
11303 let h = state
11304 .seat_history
11305 .entry(originals[i].seat.key.clone())
11306 .or_default();
11307 h.failed.insert(specs[i].id.clone());
11308 h.last_fail = Some(cur.clone());
11309 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11310 h.last_ok = None;
11311 }
11312 if let Err(e) = state.save() {
11315 tracing::warn!("could not persist a seat's failure history: {e:#}");
11316 }
11317 }
11318 if let Some(next) = next {
11319 record_handover(
11320 state,
11321 ctx.node,
11322 &originals[i].seat.key,
11323 &specs[i].id,
11324 &next.id,
11325 &cur,
11326 &fail_reason(&out),
11327 );
11328 tried[i].insert(next.id.clone());
11329 prev[i] = Some(cur.clone());
11330 seats[i] =
11331 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11332 specs[i] = next;
11333 fresh[i] = Some(cur.stem_word().to_owned());
11334 nudges[i] = 0;
11335 done[i] = Some(Err(anyhow::anyhow!(
11336 "handed over after: {}",
11337 fail_reason(&out)
11338 )));
11339 still.push(i);
11340 continue;
11341 }
11342 }
11343 if carry
11344 && !nudge_first
11345 && let Some(cur) = class.clone()
11346 {
11347 let h = state
11349 .seat_history
11350 .entry(originals[i].seat.key.clone())
11351 .or_default();
11352 h.failed.insert(specs[i].id.clone());
11353 h.last_fail = Some(cur);
11354 }
11355 let parsed = match out {
11356 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11357 AgentOutcome::Quota(o) => {
11358 last_quota[i] = o.quota.clone();
11359 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11360 }
11361 AgentOutcome::Dropped(o) => {
11365 let why = o
11366 .dropped
11367 .as_ref()
11368 .map(|d| d.why.as_str())
11369 .unwrap_or("the CLI ended the stream without delivering its answer");
11370 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11371 }
11372 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11373 Err(anyhow::anyhow!(e))
11374 }
11375 };
11376 let quota = class == Some(FailClass::Quota);
11377 let failed = parsed.is_err();
11378 if !failed && matches!(ctx.node, "judge" | "vote") {
11379 state
11382 .quota
11383 .retain(|loss| loss.seat != originals[i].seat.key);
11384 }
11385 done[i] = Some(parsed);
11386 if carry && !failed {
11387 let h = state
11388 .seat_history
11389 .entry(originals[i].seat.key.clone())
11390 .or_default();
11391 h.failed.remove(&specs[i].id);
11392 h.last_ok = Some(specs[i].id.clone());
11393 h.last_fail = None;
11394 }
11395 let agent_failure = class.is_some()
11403 && !nudge_first
11404 && !roster.is_empty()
11405 && next_agent(i, &tried[i], &specs).is_some();
11406 if failed && !quota && !agent_failure && nudges[i] < retries {
11407 nudges[i] += 1;
11408 still.push(i);
11409 }
11410 }
11411 pending = still;
11412 }
11413
11414 for (i, q) in last_quota.into_iter().enumerate() {
11417 if let Some(quota) = q {
11418 losses.push(QuotaLoss::from_report(
11419 originals[i].seat.key.clone(),
11420 ctx.node,
11421 Some("a),
11422 ));
11423 }
11424 }
11425
11426 if state.status != RunStatus::Stalled
11429 && ctx.node != "deliberate"
11430 && n > 0
11431 && losses.len() == n
11432 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11433 {
11434 block_machine_outage(state, ctx.node, losses);
11435 state.quota.append(losses);
11436 }
11437
11438 seats
11439 .into_iter()
11440 .zip(done)
11441 .zip(nudges)
11442 .map(|((seat, res), attempts)| {
11443 (
11444 seat,
11445 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11446 attempts,
11447 )
11448 })
11449 .collect()
11450}
11451
11452pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11455 let mut auth: Vec<_> = losses
11456 .iter()
11457 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11458 .map(|loss| loss.agent.clone())
11459 .collect();
11460 auth.sort();
11461 auth.dedup();
11462 let (kind, detail) = if auth.is_empty() {
11463 (
11464 "agent usage limit",
11465 "All seats reached their agent usage limits; resume after usage is available"
11466 .to_owned(),
11467 )
11468 } else {
11469 (
11470 "agent authentication",
11471 format!(
11472 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11473 auth.join(", ")
11474 ),
11475 )
11476 };
11477 state.resource_block = Some(crate::run::ResourceBlock {
11478 node: node.to_owned(),
11479 kind: kind.to_owned(),
11480 cause: crate::run::ResourceBlockCause::Other,
11481 detail: detail.clone(),
11482 at: Timestamp::now(),
11483 });
11484 state.event(node, detail);
11485}
11486
11487fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11489 results
11490 .iter()
11491 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11492}
11493
11494fn block_wave(
11498 jobs: Vec<SeatJob>,
11499 state: &mut RunState,
11500 node: &str,
11501 attempt: usize,
11502 why: &str,
11503) -> Vec<(usize, SeatState, AgentOutcome)> {
11504 state.resource_wait = None;
11505 state
11506 .active
11507 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11508 if let Err(e) = state.save() {
11509 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11510 }
11511 jobs.into_iter()
11512 .enumerate()
11513 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11514 .collect()
11515}
11516
11517async fn acquire_cache_lease(
11530 state: &mut RunState,
11531 cache_dir: &Path,
11532 owner: &crate::cache::Owner,
11533 budget: Duration,
11534 node: &str,
11535 context: &str,
11536 cancel: &(dyn Fn() -> bool + Sync),
11537) -> Result<Option<crate::cache::Guard>> {
11538 let home = crate::run::home();
11539 let started = Instant::now();
11540 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11541 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11542 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11543 Err(e) => {
11544 state.event(
11545 "verify",
11546 format!("{context}: could not check the shared build cache: {e:#}"),
11547 );
11548 if let Err(e2) = state.save() {
11549 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11550 }
11551 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11552 }
11553 };
11554 state.event(
11555 "verify",
11556 format!(
11557 "{context}: waiting for the shared build cache at {} ({})",
11558 cache_dir.display(),
11559 busy.describe()
11560 ),
11561 );
11562 state.resource_wait = Some(crate::run::ResourceWait {
11565 node: context.to_owned(),
11566 kind: "shared build cache".to_owned(),
11567 holder: busy.describe(),
11568 since: Timestamp::now(),
11569 limit_secs: budget.as_secs(),
11570 });
11571 if let Err(e) = state.save() {
11572 tracing::warn!("could not persist a cache wait: {e:#}");
11573 }
11574 let remaining = budget.saturating_sub(started.elapsed());
11575 let waited = crate::cache::wait_for_unless(
11576 &home,
11577 cache_dir,
11578 owner,
11579 remaining,
11580 Duration::from_secs(5),
11581 cancel,
11582 )
11583 .await;
11584 state.resource_wait = None;
11585 let cause = match &waited {
11586 Ok(Some(_)) => None,
11587 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11588 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11589 Some(crate::run::ResourceBlockCause::Timeout {
11590 waited_secs: started.elapsed().as_secs(),
11591 limit_secs: budget.as_secs(),
11592 })
11593 }
11594 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11595 };
11596 if let Some(cause) = cause {
11597 let detail = match &waited {
11598 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11599 Err(error) => format!("{error:#}"),
11600 };
11601 state.resource_block = Some(crate::run::ResourceBlock {
11602 cause,
11603 node: node.to_owned(),
11604 kind: "shared build cache".into(),
11605 detail,
11606 at: Timestamp::now(),
11607 });
11608 state.event(
11609 "resource",
11610 state.resource_block.as_ref().unwrap().description(),
11611 );
11612 }
11613 if let Err(error) = state.save() {
11614 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11615 }
11616 waited
11617}
11618
11619#[derive(Debug)]
11624struct CacheCheckError(String);
11625
11626impl std::fmt::Display for CacheCheckError {
11627 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11628 f.write_str(&self.0)
11629 }
11630}
11631
11632impl std::error::Error for CacheCheckError {}
11633
11634fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11636 state
11637 .resource_block
11638 .get_or_insert_with(|| crate::run::ResourceBlock {
11639 cause: crate::run::ResourceBlockCause::Other,
11640 node: node.to_owned(),
11641 kind: "shared build cache".into(),
11642 detail: format!("{error:#}"),
11643 at: Timestamp::now(),
11644 });
11645 if let Err(error) = state.save() {
11646 tracing::warn!("could not persist cache resource block: {error:#}");
11647 }
11648}
11649
11650#[allow(clippy::too_many_arguments)]
11671async fn with_cache_lease<'s, F, Fut>(
11672 state: &'s mut RunState,
11673 cache_dir: Option<&Path>,
11674 node: &str,
11675 seat: &str,
11676 worktree: &Path,
11677 head: &str,
11678 budget: Duration,
11679 context: &str,
11680 wait: &crate::repo_lock::WaitPolicy,
11681 body: F,
11682) -> Result<(Vec<CommandOutcome>, bool)>
11683where
11684 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11685 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11686{
11687 let started = Instant::now();
11690 let home = crate::run::home();
11691 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11692 let guard = if let Some(cache_dir) = cache_dir {
11693 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11694 match acquire_cache_lease(
11695 state,
11696 cache_dir,
11697 &owner,
11698 resource_limit,
11699 node,
11700 context,
11701 &|| wait.cancelled(),
11702 )
11703 .await
11704 .and_then(|g| {
11705 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11706 }) {
11707 Ok(g) => Some(g),
11708 Err(e) => {
11709 record_cache_error(state, node, &e);
11710 return Ok((
11711 vec![CommandOutcome {
11712 command: "(waiting for the shared build cache)".into(),
11713 code: None,
11714 output_tail: e.to_string(),
11715 duration_ms: started.elapsed().as_millis() as u64,
11716 resource_blocked: true,
11717 }],
11718 false,
11719 ));
11720 }
11721 }
11722 } else {
11723 None
11724 };
11725 let _coordination = coordinate_operation(state, worktree, wait).await?;
11726 if let Some(cache_dir) = cache_dir {
11727 let identity = crate::cache::Identity::new(worktree, head);
11728 if let Err(e) = crate::cache::ensure_fresh(
11729 &home,
11730 cache_dir,
11731 &identity,
11732 &state.config.verify.cargo_manifests(),
11733 ) {
11734 record_cache_error(state, node, &e);
11735 state.event(
11736 "verify",
11737 format!(
11738 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11739 worktree.display(),
11740 short(head)
11741 ),
11742 );
11743 if let Some(guard) = guard {
11744 guard.release();
11745 }
11746 return Ok((
11747 vec![CommandOutcome {
11748 command: "(confirming the shared build cache is fresh)".into(),
11749 code: None,
11750 output_tail: e.to_string(),
11751 duration_ms: started.elapsed().as_millis() as u64,
11752 resource_blocked: true,
11753 }],
11754 false,
11755 ));
11756 }
11757 }
11758 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11759 if !timed_out_pids.is_empty() {
11760 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11761 }
11762 drop(_coordination);
11765 if let Some(guard) = guard {
11766 guard.release();
11767 }
11768 Ok((outcomes, retried))
11769}
11770
11771async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11783 wait_for_pids_with(
11784 pids,
11785 crate::proc::pid_alive,
11786 LEASE_RELEASE_POLL,
11787 LEASE_RELEASE_MAX_WAIT,
11788 )
11789 .await;
11790}
11791
11792async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11798 pids: &[u32],
11799 alive: F,
11800 poll: Duration,
11801 max_wait: Duration,
11802) {
11803 let deadline = Instant::now() + max_wait;
11804 loop {
11805 if pids.iter().all(|&pid| !alive(pid)) {
11806 return;
11807 }
11808 if Instant::now() >= deadline {
11809 return;
11810 }
11811 tokio::time::sleep(poll).await;
11812 }
11813}
11814
11815fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11822 outcomes.iter().any(|o| o.resource_blocked)
11823}
11824
11825pub(crate) enum GateFix {
11827 Retry,
11829 Stop,
11832 Defer,
11835}
11836
11837pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11845 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11846 red.peek().is_some()
11847 && red.all(|o| {
11848 !o.resource_blocked
11849 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11850 && !o.output_tail.trim().is_empty()
11851 })
11852}
11853
11854fn e2e_outcome_label(o: &CommandOutcome) -> String {
11858 if o.ok() {
11859 return "pass".to_owned();
11860 }
11861 let reason = if o.build_failed() {
11862 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11863 } else {
11864 format!("FAIL ({:?})", o.code)
11865 };
11866 format!(
11867 "{reason}\n{}",
11868 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11869 )
11870}
11871
11872async fn run_e2e_with_retry(
11880 state: &mut RunState,
11881 shell: &[String],
11882 commands: &[String],
11883 worktree: &Path,
11884 timeout: Duration,
11885 context: &str,
11886) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11887 let (mut e2e, mut timed_out_pids) = run_commands(
11888 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11889 )
11890 .await;
11891 for o in &e2e {
11892 state.event(
11893 "verify",
11894 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11895 );
11896 }
11897 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11901 if verify_retried {
11902 state.event(
11903 "verify",
11904 format!(
11905 "{context}: verify could not build/link, not a test result — retrying once \
11906 before concluding"
11907 ),
11908 );
11909 let retried = run_commands(
11910 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11911 )
11912 .await;
11913 e2e = retried.0;
11914 timed_out_pids.extend(retried.1);
11917 for o in &e2e {
11918 state.event(
11919 "verify",
11920 format!(
11921 "{context}: retry `{}` -> {}",
11922 o.command,
11923 e2e_outcome_label(o)
11924 ),
11925 );
11926 }
11927 }
11928 (e2e, verify_retried, timed_out_pids)
11929}
11930
11931#[allow(clippy::too_many_arguments)]
11947async fn run_commands(
11948 state: &mut RunState,
11949 node: &str,
11950 task: &str,
11951 attempt: usize,
11952 shell: &[String],
11953 commands: &[String],
11954 cwd: &Path,
11955 timeout: Duration,
11956) -> (Vec<CommandOutcome>, Vec<u32>) {
11957 if commands.is_empty() {
11958 return (Vec::new(), Vec::new());
11963 }
11964 let mut out = Vec::new();
11965 let mut timed_out_pids = Vec::new();
11966 let total = commands.len();
11967 for (idx, command) in commands.iter().enumerate() {
11968 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11969 if let Err(e) = state.save() {
11970 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11971 }
11972 let started = Instant::now();
11973 let mut cmd = tokio::process::Command::new(&shell[0]);
11974 cmd.quiet();
11975 for key in crate::worker::INVOCATION_ENV {
11979 cmd.env_remove(key);
11980 }
11981 cmd.args(&shell[1..])
11982 .arg(command)
11983 .current_dir(cwd)
11984 .stdin(std::process::Stdio::null())
11985 .stdout(std::process::Stdio::piped())
11986 .stderr(std::process::Stdio::piped())
11987 .kill_on_drop(true);
11988 let mut outputs = Vec::new();
11989 let protected = (|| -> Result<()> {
11990 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11991 outputs.push(guard);
11992 }
11993 if let Some(cache) = state.config.cache_dir() {
11994 outputs.push(crate::retention::OutputLease::cache(
11995 &crate::run::home(),
11996 &cache,
11997 false,
11998 )?);
11999 }
12000 Ok(())
12001 })();
12002 if let Err(e) = protected {
12003 out.push(CommandOutcome {
12004 command: command.clone(),
12005 code: None,
12006 output_tail: format!("output ownership unavailable: {e:#}"),
12007 duration_ms: 0,
12008 resource_blocked: true,
12009 });
12010 continue;
12011 }
12012 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12013 let (code, body) = match spawned {
12014 Ok(mut running) => {
12015 let pid = running.child().id();
12016 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12017 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12018 let mut paths = vec![cwd.to_owned()];
12019 if let Some(cache) = state.config.cache_dir() {
12020 paths.push(cache);
12021 }
12022 let mut pressure = None;
12023 let status = tokio::select! {
12024 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12025 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12026 pressure = Some(reason); None
12027 }
12028 };
12029 let (code, failure) = match status {
12030 Some(Ok(Ok(status))) => (
12031 status.code(),
12032 status
12033 .code()
12034 .is_none()
12035 .then(|| format!("command terminated: {status}")),
12036 ),
12037 Some(Ok(Err(error))) => {
12038 (None, Some(format!("failed waiting for command: {error}")))
12039 }
12040 Some(Err(_)) => {
12041 if !running.coordinated()
12045 && let Some(pid) = pid
12046 {
12047 timed_out_pids.push(pid);
12048 }
12049 (
12050 None,
12051 Some(format!("timed out after {}s", timeout.as_secs())),
12052 )
12053 }
12054 None => (None, None),
12055 };
12056 running.finish().await;
12057 if let Some(reason) = pressure {
12058 state.resource_block = Some(crate::run::ResourceBlock {
12059 cause: crate::run::ResourceBlockCause::Other,
12060 node: node.to_owned(),
12061 kind: "disk pressure".to_owned(),
12062 detail: reason.clone(),
12063 at: Timestamp::now(),
12064 });
12065 out.push(CommandOutcome {
12066 command: command.clone(),
12067 code: None,
12068 output_tail: reason,
12069 duration_ms: started.elapsed().as_millis() as u64,
12070 resource_blocked: true,
12071 });
12072 break;
12073 }
12074 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12075 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12076 if let Some(failure) = failure {
12077 body.push('\n');
12078 body.push_str(&failure);
12079 }
12080 (code, body)
12081 }
12082 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12083 };
12084 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12085 if !body.is_empty() {
12086 match crate::run::write_verification_artifact(state, task, &body) {
12090 Ok(name) if summary != body => {
12091 summary = format!("[full output: artifacts/{name}]\n{summary}");
12092 }
12093 Ok(_) => {}
12094 Err(e) => {
12095 let warning = format!("could not save the full {task} output: {e:#}");
12096 tracing::warn!("{warning}");
12097 state.event(node, warning);
12098 if summary != body {
12099 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12100 }
12101 }
12102 }
12103 }
12104 out.push(CommandOutcome {
12105 command: command.clone(),
12106 code,
12107 output_tail: summary,
12108 duration_ms: started.elapsed().as_millis() as u64,
12109 resource_blocked: false,
12110 });
12111 }
12112 state.task_finished(task);
12113 if let Err(e) = state.save() {
12114 tracing::warn!("could not persist the end of {task}: {e:#}");
12115 }
12116 (out, timed_out_pids)
12117}
12118
12119fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12131 let repo = repo.display();
12132 match style {
12133 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12134 MergeStyle::Squash => {
12135 let subject = message
12138 .lines()
12139 .next()
12140 .unwrap_or(branch)
12141 .replace(['\\', '"', '$', '`'], "");
12142 format!(
12143 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12144 )
12145 }
12146 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12147 }
12148}
12149
12150const PR_TITLE_MAX: usize = 240;
12161
12162fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12173 let line = queue::first_line(text)?;
12174 let chars: Vec<char> = line.chars().collect();
12175 let (kept, mark) = if chars.len() <= max {
12176 (line.to_owned(), "")
12177 } else if let Some(end) = sentence_end(&chars, max) {
12178 (chars[..end].iter().collect::<String>(), "")
12179 } else {
12180 let room = max.saturating_sub(3);
12181 let cut = if chars[room].is_whitespace() {
12184 room
12185 } else {
12186 chars[..room]
12187 .iter()
12188 .rposition(|c| c.is_whitespace())
12189 .unwrap_or(room)
12190 };
12191 let head: String = chars[..cut].iter().collect();
12192 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12193 (head.to_owned(), "...")
12194 };
12195 if kept.is_empty() {
12196 return None;
12197 }
12198 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12199 return None;
12200 }
12201 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12202 return None;
12203 }
12204 Some(format!("{kept}{mark}"))
12205}
12206
12207fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12211 const MIN: usize = 20;
12212 for i in MIN..max.min(chars.len()) {
12213 if !matches!(chars[i], '.' | '!' | '?') {
12214 continue;
12215 }
12216 let Some(&next) = chars.get(i + 1) else {
12217 continue;
12218 };
12219 if !next.is_whitespace() {
12220 continue;
12221 }
12222 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12223 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12224 continue;
12225 }
12226 let word: String = chars[..i]
12227 .iter()
12228 .rev()
12229 .take_while(|c| !c.is_whitespace())
12230 .collect::<Vec<_>>()
12231 .into_iter()
12232 .rev()
12233 .collect();
12234 let word = word.to_ascii_lowercase();
12235 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12236 continue;
12237 }
12238 let end = chars[..i]
12239 .iter()
12240 .rposition(|c| !c.is_whitespace())
12241 .map_or(i, |p| p + 1);
12242 return Some(end);
12243 }
12244 None
12245}
12246
12247struct PrMessage {
12252 title: String,
12253 body: String,
12254}
12255
12256impl PrMessage {
12257 fn commit_message(&self) -> String {
12261 format!("{}\n\n{}", self.title, self.body)
12262 }
12263}
12264
12265fn title_marker(line: &str) -> Option<&str> {
12267 let line = line.trim();
12268 let head = line.get(..6)?;
12269 head.eq_ignore_ascii_case("title:")
12270 .then(|| line[6..].trim())
12271}
12272
12273fn summary_title(summary: &str) -> Option<String> {
12278 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12279 let raw = title_marker(first)?;
12280 if raw.is_empty() {
12281 return None;
12282 }
12283 let title = queue::title_from(raw, PR_TITLE_MAX);
12284 let lower = title.to_ascii_lowercase();
12285 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12286 return None;
12287 }
12288 Some(title)
12289}
12290
12291const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12293
12294pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12296
12297pub(crate) fn is_review_run(state: &RunState) -> bool {
12301 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12302}
12303
12304fn review_title(state: &RunState) -> Option<String> {
12309 english_subject(
12310 state.reviewed_commits.as_ref()?.first()?,
12311 state.config.graph.github_english(),
12312 )
12313}
12314
12315fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12318 let raw = raw.trim();
12319 if raw.is_empty() {
12320 return None;
12321 }
12322 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12323 return None;
12324 }
12325 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12326 return None;
12327 }
12328 let title = queue::title_from(raw, PR_TITLE_MAX);
12329 let lower = title.to_ascii_lowercase();
12330 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12331 return None;
12332 }
12333 Some(title)
12334}
12335
12336#[derive(Debug, Clone, PartialEq, Eq)]
12339struct BranchFacts {
12340 commits: Vec<(String, String)>,
12342 stat: String,
12344}
12345
12346const STAT_FILE_LINES: usize = 25;
12348const COMMITS_MAX_CHARS: usize = 20_000;
12350
12351async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12354 let commits = git::commit_log(repo, base, branch).await.ok()?;
12355 if commits.is_empty() {
12356 return None;
12357 }
12358 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12359 let lines: Vec<&str> = stat.lines().collect();
12360 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12361 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12362 let more = format!(" ... {omitted} more file(s)");
12363 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12364 kept.push(&more);
12365 kept.push(lines[lines.len() - 1]);
12366 kept.join("\n")
12367 } else {
12368 lines.join("\n")
12369 };
12370 Some(BranchFacts { commits, stat })
12371}
12372
12373fn markdown_safe(text: &str) -> String {
12376 text.replace("</details>", "</details>")
12377 .replace("\x60\x60\x60", "~~~")
12378}
12379
12380fn neutral_title(state: &RunState, winner: char) -> String {
12381 format!(
12382 "chore: land candidate {} of run {}",
12383 winner.to_ascii_uppercase(),
12384 state.id
12385 )
12386}
12387
12388pub(crate) async fn repair_landing_gate(
12391 state: &mut RunState,
12392 winner: &Candidate,
12393 outcomes: &[CommandOutcome],
12394) -> Result<GateFix> {
12395 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12396 return Ok(GateFix::Stop);
12397 }
12398 let mut runner = Runner {
12399 roles: state.config.resolve_roles()?,
12400 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12401 state: state.clone(),
12402 pause: Pause::new(),
12403 interrupt: Pause::new(),
12404 };
12405 let result = runner.gate_fix_round(winner, outcomes).await;
12406 *state = runner.state;
12407 result
12408}
12409
12410pub(crate) async fn verify_current_head(
12413 state: &mut RunState,
12414 winner: &Candidate,
12415 wait: &crate::repo_lock::WaitPolicy,
12416) -> Result<bool> {
12417 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12418 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12420 state
12421 .verification_history
12422 .push(crate::run::VerificationEvidence {
12423 superseded_by: Some(head.clone()),
12424 archived_at: Some(Timestamp::now()),
12425 fixes_spent: Some(state.gate_fixes.len()),
12426 gate_head: None,
12427 gate_ran: false,
12428 gate: Vec::new(),
12429 e2e: state.head_e2e.take(),
12430 });
12431 }
12432 if let Some(e) = &state.head_e2e
12433 && e.head == head
12434 {
12435 if e.passed {
12436 return Ok(true);
12437 }
12438 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12439 state.status = RunStatus::Blocked;
12440 state.save()?;
12441 return Ok(false);
12442 }
12443 } else if state.reviews.last().is_some_and(|r| {
12444 r.verified_head.as_deref() == Some(head.as_str())
12445 && (r.e2e_status() == E2eStatus::Passed
12446 || (r.e2e_status() == E2eStatus::NotConfigured
12447 && state.config.verify.e2e.is_empty()))
12448 }) || state
12449 .gate_repair_e2e
12450 .as_ref()
12451 .is_some_and(|e| e.head == head && e.passed)
12452 {
12453 return Ok(true);
12454 }
12455 if let Some(e) = &state.gate_repair_e2e
12456 && e.head == head
12457 && !e.passed
12458 && !e.outcomes.is_empty()
12459 && !verify_inconclusive(&e.outcomes)
12460 {
12461 state.status = RunStatus::Blocked;
12462 state.save()?;
12463 return Ok(false);
12464 }
12465 state.head_e2e = Some(crate::run::GateRepairE2e {
12466 head: head.clone(),
12467 passed: false,
12468 outcomes: Vec::new(),
12469 });
12470 state.save()?;
12471 let commands = state.config.verify.e2e.clone();
12472 let shell = state.config.shell();
12473 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12474 let cache_dir = state.config.cache_dir();
12475 let outcomes = if commands.is_empty() {
12476 Vec::new()
12477 } else {
12478 let (outcomes, _) = with_cache_lease(
12479 state,
12480 cache_dir.as_deref(),
12481 "e2e",
12482 "e2e",
12483 &winner.worktree,
12484 &head,
12485 timeout,
12486 "current head e2e",
12487 wait,
12488 |state, budget| async move {
12489 run_e2e_with_retry(
12490 state,
12491 &shell,
12492 &commands,
12493 &winner.worktree,
12494 budget,
12495 "current head e2e",
12496 )
12497 .await
12498 },
12499 )
12500 .await?;
12501 outcomes
12502 };
12503 let unresolved = verify_inconclusive(&outcomes);
12504 let passed = !unresolved
12505 && outcomes.iter().all(CommandOutcome::ok)
12506 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12507 if !outcomes.is_empty() && !unresolved {
12508 state.record_base_progress(&head);
12509 }
12510 state.head_e2e = Some(crate::run::GateRepairE2e {
12511 head: head.clone(),
12512 passed,
12513 outcomes,
12514 });
12515 if !passed && !unresolved {
12516 state.status = RunStatus::Blocked;
12517 state.event(
12518 "e2e",
12519 format!("verify.e2e failed on head {head}; nothing published"),
12520 );
12521 }
12522 state.save()?;
12523 Ok(passed)
12524}
12525
12526pub(crate) async fn check_gate_repair_e2e(
12529 state: &mut RunState,
12530 winner: &Candidate,
12531 wait: &crate::repo_lock::WaitPolicy,
12532) -> Result<GateFix> {
12533 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12534 return Ok(GateFix::Retry);
12535 }
12536 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12537 if state
12538 .gate_repair_e2e
12539 .as_ref()
12540 .is_some_and(|r| r.head == head && r.passed)
12541 {
12542 return Ok(GateFix::Retry);
12543 }
12544 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12545 head: head.clone(),
12546 passed: false,
12547 outcomes: Vec::new(),
12548 });
12549 state.save()?;
12550 let commands = state.config.verify.e2e.clone();
12551 let outcomes = if commands.is_empty() {
12552 Vec::new()
12553 } else {
12554 let shell = state.config.shell();
12555 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12556 let cache_dir = state.config.cache_dir();
12557 let (outcomes, _) = with_cache_lease(
12558 state,
12559 cache_dir.as_deref(),
12560 "e2e",
12561 "e2e",
12562 &winner.worktree,
12563 &head,
12564 timeout,
12565 "gate repair e2e",
12566 wait,
12567 |state, budget| async move {
12568 run_e2e_with_retry(
12569 state,
12570 &shell,
12571 &commands,
12572 &winner.worktree,
12573 budget,
12574 "gate repair e2e",
12575 )
12576 .await
12577 },
12578 )
12579 .await?;
12580 outcomes
12581 };
12582 let deferred = verify_inconclusive(&outcomes);
12583 let passed = outcomes.iter().all(CommandOutcome::ok)
12584 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12585 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12586 head,
12587 passed,
12588 outcomes,
12589 });
12590 state.save()?;
12591 if deferred {
12592 Ok(GateFix::Defer)
12593 } else if !passed {
12594 state.event(
12595 "gate",
12596 "verify.e2e failed after gate repair; nothing published",
12597 );
12598 Ok(GateFix::Stop)
12599 } else {
12600 Ok(GateFix::Retry)
12601 }
12602}
12603
12604pub(crate) async fn run_landing_gate(
12606 state: &mut RunState,
12607 winner: &Candidate,
12608 wait: &crate::repo_lock::WaitPolicy,
12609) -> Result<Vec<CommandOutcome>> {
12610 let shell = state.config.shell();
12611 let gate_commands = state.config.verify.gate.clone();
12612 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12613 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12614 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12615 Vec::new()
12616 } else {
12617 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12618 let cache_dir = state.config.cache_dir();
12619 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12620 let (outcomes, _) = with_cache_lease(
12621 state,
12622 cache_dir.as_deref(),
12623 "gate",
12624 "gate",
12625 &winner.worktree,
12626 &head,
12627 timeout,
12628 "final gate",
12629 wait,
12630 |state, budget| {
12631 let shell = shell.clone();
12632 let gate_commands = gate_commands.clone();
12633 let winner = winner.clone();
12634 async move {
12635 Runner::run_pre_gate(state, &winner).await;
12636 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12638 let (outcomes, timed_out_pids) = run_commands(
12639 state,
12640 "gate",
12641 "gate",
12642 0,
12643 &shell,
12644 &gate_commands,
12645 &winner.worktree,
12646 budget,
12647 )
12648 .await;
12649 (outcomes, false, timed_out_pids)
12650 }
12651 },
12652 )
12653 .await?;
12654 outcomes
12655 };
12656 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12657 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12658 let mut outcomes = outcomes;
12659 outcomes.push(CommandOutcome {
12660 command: "gate head stability".into(), code: None,
12661 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12662 duration_ms: 0, resource_blocked: false,
12663 });
12664 return Ok(outcomes);
12665 }
12666 if outcomes.is_empty() {
12667 state.event("gate", "no gate commands configured; not tested");
12672 }
12673 for o in &outcomes {
12674 state.event(
12675 "gate",
12676 format!(
12677 "`{}` -> {}",
12678 o.command,
12679 if o.ok() {
12680 "pass".to_owned()
12681 } else {
12682 format!(
12683 "FAIL ({:?})\n{}",
12684 o.code,
12685 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12686 )
12687 }
12688 ),
12689 );
12690 }
12691 Ok(outcomes)
12692}
12693
12694pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12700 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12701 ""
12702 } else {
12703 pr_title
12704 }
12705}
12706
12707pub fn landing_subject_source(state: &RunState) -> String {
12711 if is_review_run(state) {
12712 let winner = state.candidates.first().map_or('A', |c| c.label);
12713 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12714 }
12715 state.instruction.clone()
12716}
12717
12718fn summary_without_title(summary: &str) -> String {
12721 let mut lines = summary.trim().lines().peekable();
12722 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12723 lines.next();
12724 }
12725 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12726}
12727
12728#[cfg(test)]
12740fn pr_message(state: &RunState, winner: char) -> PrMessage {
12741 pr_message_with(state, winner, None)
12742}
12743
12744#[cfg(test)]
12747fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12748 let raw = pr_message_raw(
12749 state,
12750 winner,
12751 facts,
12752 crate::pr_outcome::FinalHead::Unreported,
12753 ChangeNote::NotAsked,
12754 );
12755 let id = crate::scrub::Identity::current();
12756 PrMessage {
12757 title: crate::scrub::scrub(&raw.title, &id),
12758 body: crate::scrub::scrub(&raw.body, &id),
12759 }
12760}
12761
12762#[derive(Debug, Clone, Copy)]
12764enum ChangeNote<'a> {
12765 NotAsked,
12767 Summary(&'a crate::run::ChangeSummary),
12769 Unavailable(&'a str),
12771}
12772
12773fn pr_message_raw(
12774 state: &RunState,
12775 winner: char,
12776 facts: Option<&BranchFacts>,
12777 head: crate::pr_outcome::FinalHead<'_>,
12778 change: ChangeNote<'_>,
12779) -> PrMessage {
12780 if !matches!(change, ChangeNote::NotAsked) {
12781 let (title, mut body) = match change {
12782 ChangeNote::Summary(c) => (
12783 c.title.clone(),
12784 format!("## Summary\n\n{}\n\n", c.body.trim()),
12785 ),
12786 ChangeNote::Unavailable(why) => (
12787 neutral_title(state, winner),
12788 format!(
12789 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12790 ),
12791 ),
12792 ChangeNote::NotAsked => unreachable!(),
12793 };
12794 body.push_str(&public_pr_evidence(state, head));
12795 body.push_str(&format!(
12796 "\n---\nmagi:run/{} magi:candidate-{}\n",
12797 state.id,
12798 winner.to_ascii_lowercase()
12799 ));
12800 return PrMessage { title, body };
12801 }
12802 pr_preview(state, winner, facts, head)
12803}
12804
12805fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12808 let mut body = crate::pr_outcome::review_rounds(state);
12809 body.push_str(&crate::pr_outcome::final_verification(state, head));
12810 let open = state.last_round_findings();
12811 if !open.is_empty() {
12812 body.push_str("## Open review findings\n\n");
12813 for f in open {
12814 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12815 }
12816 body.push('\n');
12817 }
12818 if let Some(c) = state.change_summary.as_ref()
12819 && head == crate::pr_outcome::FinalHead::At(&c.head)
12820 {
12821 body.push_str(&format!(
12822 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12823 c.base, c.head
12824 ));
12825 }
12826 body
12827}
12828
12829fn pr_preview(
12831 state: &RunState,
12832 winner: char,
12833 facts: Option<&BranchFacts>,
12834 head: crate::pr_outcome::FinalHead<'_>,
12835) -> PrMessage {
12836 let summary = state
12837 .candidates
12838 .iter()
12839 .find(|c| c.label == winner)
12840 .map(|c| c.summary.as_str())
12841 .unwrap_or_default();
12842 let review = is_review_run(state);
12845 let title = if review {
12846 facts
12847 .and_then(|f| {
12848 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12849 })
12850 .or_else(|| review_title(state))
12851 .or_else(|| {
12852 state
12853 .candidates
12854 .iter()
12855 .find(|c| c.label == winner)
12856 .filter(|c| !c.branch.starts_with("magi/"))
12857 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12858 })
12859 .unwrap_or_else(|| neutral_title(state, winner))
12860 } else {
12861 summary_title(summary).unwrap_or_else(|| {
12862 english_title(
12863 &state.instruction,
12864 PR_TITLE_MAX,
12865 state.config.graph.github_english(),
12866 )
12867 .unwrap_or_else(|| neutral_title(state, winner))
12868 })
12869 };
12870
12871 let mut body = String::new();
12872 let what = summary_without_title(summary);
12873 if !what.is_empty() {
12874 body.push_str("## Summary\n\n");
12875 body.push_str(&what);
12876 body.push_str("\n\n");
12877 }
12878
12879 body.push_str(&crate::pr_outcome::review_rounds(state));
12881 body.push_str(&crate::pr_outcome::final_verification(state, head));
12882 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12883
12884 let open = state.open_findings();
12885 if !open.is_empty() {
12886 body.push_str("## Open review findings\n\n");
12887 for f in &open {
12888 body.push_str(&format!(
12889 "- `{}` [{:?}] {}\n",
12890 f.id, f.severity, "Context retained in local review records."
12891 ));
12892 }
12893 body.push('\n');
12894 }
12895
12896 if let Some(fix) = fix
12897 && !fix.rejected.is_empty()
12898 {
12899 body.push_str("## Declined by the fixer\n\n");
12900 for r in &fix.rejected {
12901 body.push_str(&format!(
12902 "- `{}`: {}\n",
12903 r.id, "Reason retained in local review records."
12904 ));
12905 }
12906 body.push('\n');
12907 }
12908
12909 if review {
12910 body.push_str("## Commits under review\n\n");
12912 if let Some(facts) = facts {
12913 let mut left = COMMITS_MAX_CHARS;
12914 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12915 let mut entry = format!("- {}\n", markdown_safe(subject));
12916 for l in markdown_safe(text).lines() {
12917 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12918 }
12919 if left == 0 {
12920 body.push_str(&format!(
12921 "- ... {} more commit(s)\n",
12922 facts.commits.len() - i
12923 ));
12924 break;
12925 }
12926 if entry.len() > left {
12927 let mut end = left;
12930 while !entry.is_char_boundary(end) {
12931 end -= 1;
12932 }
12933 entry.truncate(end);
12934 entry.push_str("\n ... (truncated)\n");
12935 left = 0;
12936 } else {
12937 left -= entry.len();
12938 }
12939 body.push_str(&entry);
12940 }
12941 if !facts.stat.trim().is_empty() {
12942 body.push_str(&format!(
12943 "\n## Diff stat\n\n```\n{}\n```\n",
12944 markdown_safe(facts.stat.trim())
12945 ));
12946 }
12947 } else {
12948 match &state.reviewed_commits {
12949 Some(subjects) => {
12950 for s in subjects {
12951 body.push_str(&format!("- {}\n", s.trim()));
12952 }
12953 }
12954 None => {
12955 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12958 body.push_str(rest.trim());
12959 body.push('\n');
12960 }
12961 }
12962 }
12963 } else {
12964 let task = state.instruction.trim();
12965 let task = if task.is_empty() {
12966 "(empty task)"
12967 } else {
12968 task
12969 };
12970 body.push_str(&format!(
12971 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12972 task.replace("</details>", "</details>")
12973 ));
12974 }
12975
12976 body.push_str(&format!(
12977 "\n---\nmagi:run/{} magi:candidate-{}\n",
12978 state.id,
12979 winner.to_ascii_lowercase()
12980 ));
12981
12982 PrMessage { title, body }
12983}
12984
12985fn seeded_instruction(state: &RunState) -> String {
12988 let mut inputs = Vec::new();
12989 if let Some(start) = &state.sources.start_from {
12990 inputs.push(format!(
12991 "Start from `{}` at `{}` (including ancestry).",
12992 start.token, start.sha
12993 ));
12994 }
12995 for pick in &state.sources.pick_commits {
12996 inputs.push(format!(
12997 "Pick `{}` at `{}` (single commit).",
12998 pick.token, pick.sha
12999 ));
13000 }
13001 if inputs.is_empty() {
13002 state.instruction.clone()
13003 } else {
13004 format!(
13005 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13006 state.instruction,
13007 inputs.join("\n")
13008 )
13009 }
13010}
13011
13012async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13016 let base = &state.base_branch;
13017 let mut against = base.clone();
13018 if mode == MergeMode::Pr {
13019 let remote = &state.config.merge.remote;
13023 let tracking = format!("{remote}/{base}");
13024 let fetched = git::fetch(repo, remote, base).await;
13025 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13026 against = tracking;
13027 } else {
13028 return false;
13029 }
13030 }
13031 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13032}
13033
13034fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13037 let mut detail = format!(
13038 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13039 nothing to open a pull request for"
13040 );
13041 detail.push_str(
13042 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13043 );
13044 detail
13045}
13046
13047#[derive(Debug, PartialEq, Eq)]
13050enum PrPlan {
13051 Create,
13052 Adopt { url: String, title: String },
13053 Stop(String),
13054}
13055
13056fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13059 match found {
13060 Ok(land::OpenPr::None) => PrPlan::Create,
13061 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13062 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13063 "several open pull requests exist for this branch, not picking one: {}",
13064 urls.join(" ")
13065 )),
13066 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13067 }
13068}
13069
13070pub(crate) async fn gh_pr_create(
13072 cwd: &Path,
13073 base: &str,
13074 head: &str,
13075 title: &str,
13076 body: &str,
13077) -> Result<String> {
13078 let out = tokio::process::Command::new("gh")
13079 .args([
13080 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13081 ])
13082 .current_dir(cwd)
13083 .quiet()
13084 .stdin(std::process::Stdio::null())
13085 .output()
13086 .await
13087 .context("spawn gh")?;
13088 if out.status.success() {
13089 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13090 } else {
13091 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13092 }
13093}
13094
13095async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13097 match std::fs::symlink_metadata(path) {
13098 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13099 git::worktree_remove_clean(repo, path).await?;
13102 return Ok(());
13103 }
13104 Err(e) => return Err(e.into()),
13105 Ok(_) => {}
13106 }
13107 if !git::is_clean(path).await? {
13108 bail!(
13109 "worktree has uncommitted work; refusing removal: {}",
13110 path.display()
13111 );
13112 }
13113 if !git::worktree_remove_clean(repo, path).await? {
13114 bail!("worktree removal refused: {}", path.display());
13115 }
13116 Ok(())
13117}
13118
13119pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13128 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13129 bail!("refusing fold: {reason}");
13130 }
13131 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13132 bail!("candidate preservation incomplete; refusing cleanup");
13133 }
13134 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13138 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13139 let _driver = crate::driver::maintenance(home, &state.id)?;
13140 let mut output_guards = Vec::new();
13141 for entry in std::fs::read_dir(state.worktree_root())
13142 .into_iter()
13143 .flatten()
13144 .flatten()
13145 {
13146 if entry.path().join(".git").is_file() {
13147 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13148 }
13149 }
13150 let repo = state.repo.clone();
13151 let wait = repository_wait(state, None);
13154 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13155 Ok(lease) => lease,
13156 Err(e) => {
13157 state.save_under(home)?;
13158 return Err(e);
13159 }
13160 };
13161 let _source_outputs = if state.status == RunStatus::Merged
13164 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13165 {
13166 verify_merged_fold_sources(state, drop_winner, home).await?
13167 } else {
13168 Vec::new()
13169 };
13170 let root = state.worktree_root();
13171 let winner = state.tally.as_ref().map(|t| t.winner);
13172 let mut removed = Vec::new();
13173
13174 for i in 0..state.candidates.len() {
13175 let c = state.candidates[i].clone();
13176 let is_winner = Some(c.label) == winner;
13177 if is_winner && !drop_winner {
13178 continue;
13179 }
13180 let existed = c.worktree.exists();
13181 remove_preserved_worktree(&repo, &c.worktree).await?;
13184 if existed {
13185 removed.push(c.worktree.to_string_lossy().into_owned());
13186 }
13187 let handed_over = state.released_branches.contains(&c.branch);
13190 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13191 if !git::branch_delete(&repo, &c.branch).await? {
13192 bail!("candidate branch removal refused");
13193 }
13194 removed.push(c.branch.clone());
13195 }
13196 state.candidates[i].folded = true;
13197 }
13198
13199 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13200 let path = name.path();
13201 let keep = !drop_winner
13202 && winner.is_some_and(|w| {
13203 path.file_name()
13204 .is_some_and(|n| n == format!("cand-{w}").as_str())
13205 });
13206 if keep {
13207 continue;
13208 }
13209 remove_preserved_worktree(&repo, &path).await?;
13210 removed.push(path.to_string_lossy().into_owned());
13211 }
13212
13213 remove_if_empty(&root);
13222
13223 if state.enabled_worktree_config && drop_winner {
13224 git::release_worktree_config(&repo).await.ok();
13228 state.enabled_worktree_config = false;
13229 }
13230 state.save_under(home)?;
13231 Ok(removed)
13232}
13233
13234async fn verify_merged_fold_sources(
13235 state: &RunState,
13236 drop_winner: bool,
13237 home: &Path,
13238) -> Result<Vec<std::fs::File>> {
13239 let winner = state.tally.as_ref().map(|t| t.winner);
13240 let mut heads = BTreeSet::new();
13241 for candidate in &state.candidates {
13242 if Some(candidate.label) == winner && !drop_winner {
13243 continue;
13244 }
13245 anyhow::ensure!(
13246 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13247 "candidate recovery source is outside the inventoried run root: {}",
13248 candidate.worktree.display()
13249 );
13250 match std::fs::symlink_metadata(&candidate.worktree) {
13251 Ok(_) => {}
13252 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13253 Err(error) => return Err(error.into()),
13254 }
13255 if !state.released_branches.contains(&candidate.branch)
13256 && git::branch_exists(&state.repo, &candidate.branch).await?
13257 {
13258 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13259 }
13260 }
13261 let mut cargo_guards = Vec::new();
13262 let entries = match std::fs::read_dir(state.worktree_root()) {
13263 Ok(entries) => Some(entries),
13264 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13265 Err(error) => return Err(error.into()),
13266 };
13267 for entry in entries.into_iter().flatten() {
13268 let path = entry?.path();
13269 let metadata = std::fs::symlink_metadata(&path)?;
13270 anyhow::ensure!(
13271 metadata.is_dir() && !metadata.file_type().is_symlink(),
13272 "non-directory or symlink recovery source retained: {}",
13273 path.display()
13274 );
13275 if !drop_winner
13276 && winner.is_some_and(|w| {
13277 path.file_name()
13278 .is_some_and(|name| name == format!("cand-{w}").as_str())
13279 })
13280 {
13281 continue;
13282 }
13283 anyhow::ensure!(
13284 path.join(".git").is_file(),
13285 "unregistered recovery source retained: {}",
13286 path.display()
13287 );
13288 anyhow::ensure!(
13289 git::git(
13290 &path,
13291 &[
13292 "status",
13293 "--porcelain",
13294 "--untracked-files=all",
13295 "--ignored",
13296 "--",
13297 ".",
13298 ":(exclude)target"
13299 ]
13300 )
13301 .await?
13302 .is_empty(),
13303 "source changes or ignored recovery files retained: {}",
13304 path.display()
13305 );
13306 match std::fs::symlink_metadata(path.join("target")) {
13307 Ok(_) => {
13308 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13309 }
13310 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13311 Err(error) => return Err(error.into()),
13312 }
13313 for checkpoint in [
13314 "rebase-merge",
13315 "rebase-apply",
13316 "sequencer",
13317 "CHERRY_PICK_HEAD",
13318 "MERGE_HEAD",
13319 "REBASE_HEAD",
13320 "REVERT_HEAD",
13321 "BISECT_LOG",
13322 "AUTO_MERGE",
13323 "MERGE_MSG",
13324 ] {
13325 let location = git::git(
13326 &path,
13327 &[
13328 "rev-parse",
13329 "--path-format=absolute",
13330 "--git-path",
13331 checkpoint,
13332 ],
13333 )
13334 .await?;
13335 anyhow::ensure!(
13336 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13337 if e.kind() == std::io::ErrorKind::NotFound),
13338 "Git recovery checkpoint retained: {checkpoint}"
13339 );
13340 }
13341 heads.insert(git::rev_parse(&path, "HEAD").await?);
13342 }
13343 crate::fold_evidence::preserve(state, home, &heads).await?;
13344
13345 Ok(cargo_guards)
13346}
13347
13348fn remove_if_empty(dir: &Path) {
13359 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13360 std::fs::remove_dir(dir).ok();
13361 }
13362}
13363
13364pub fn worst_open(state: &RunState) -> Option<Severity> {
13366 state
13367 .reviews
13368 .last()?
13369 .reviews
13370 .iter()
13371 .flat_map(|r| r.findings.iter())
13372 .map(|f| f.severity)
13373 .max()
13374}
13375
13376async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13380 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13381 git::acquire_worktree_config(repo).await?;
13382 state.enabled_worktree_config = true;
13383 }
13384 Ok(())
13385}
13386
13387#[cfg(test)]
13388mod tests {
13389 #[tokio::test]
13390 async fn launcher_base_fetch_requires_remote_success() {
13391 let tmp = tempfile::tempdir().unwrap();
13392 let repo = tmp.path().join("repo");
13393 let remote = tmp.path().join("remote");
13394 std::fs::create_dir(&repo).unwrap();
13395 std::fs::create_dir(&remote).unwrap();
13396 git::git(&remote, &["init", "--bare", "-b", "main"])
13397 .await
13398 .unwrap();
13399 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13400 git::git(
13401 &repo,
13402 &[
13403 "-c",
13404 "user.name=Fixture",
13405 "-c",
13406 "user.email=fixture@example.invalid",
13407 "commit",
13408 "--allow-empty",
13409 "-m",
13410 "fixture",
13411 ],
13412 )
13413 .await
13414 .unwrap();
13415 git::git(
13416 &repo,
13417 &["remote", "add", "origin", remote.to_str().unwrap()],
13418 )
13419 .await
13420 .unwrap();
13421 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13422 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13423 assert_eq!(
13424 resolve_base(&repo, "main", "origin", None)
13425 .await
13426 .unwrap()
13427 .commit,
13428 head
13429 );
13430 #[cfg(unix)]
13431 {
13432 let policy: crate::config::Coordination = toml::from_str(
13433 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13434 ).unwrap();
13435 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13436 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13437 let held = crate::repo_lock::SeatLease::operation_with(
13438 &repo,
13439 Some(&policy),
13440 &crate::repo_lock::WaitPolicy::default(),
13441 )
13442 .await
13443 .unwrap();
13444 assert!(held.is_some());
13445 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13446 tokio::pin!(fetch);
13447 assert!(
13448 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13449 .await
13450 .is_err()
13451 );
13452 assert!(
13453 !repo.join(".git/FETCH_HEAD").exists(),
13454 "no fetch before admission"
13455 );
13456 drop(held);
13457 let fetched = fetch.await.unwrap();
13458 assert_eq!(fetched.commit, head);
13459 assert_eq!(fetched.remote, "origin");
13460 assert_eq!(fetched.branch, "main");
13461 }
13462 git::git(
13464 &repo,
13465 &[
13466 "remote",
13467 "set-url",
13468 "origin",
13469 tmp.path().join("missing").to_str().unwrap(),
13470 ],
13471 )
13472 .await
13473 .unwrap();
13474 let error = resolve_base(&repo, "main", "origin", None)
13475 .await
13476 .unwrap_err()
13477 .to_string();
13478 assert!(error.contains("cannot read origin/main"));
13479 assert!(error.contains("refusing to branch off"));
13480 }
13481
13482 #[test]
13483 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13484 let leaked = vec!["chore(deps): update a crate".to_owned()];
13485 let own = "fix(daemon): apply a chosen action";
13486 assert!(should_retitle("", own, &leaked));
13487 assert!(should_retitle(
13488 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13489 own,
13490 &leaked
13491 ));
13492 assert!(should_retitle(
13493 "chore: land candidate A of run 1",
13494 own,
13495 &leaked
13496 ));
13497 assert!(should_retitle(
13498 "magi: candidate A (uncommitted work)",
13499 own,
13500 &leaked
13501 ));
13502 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13503 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13504 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13505 }
13506
13507 #[test]
13508 fn pr_merge_plan_creates_adopts_or_stops() {
13509 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13510 assert_eq!(
13511 pr_merge_plan(Ok(land::OpenPr::One {
13512 url: "u".into(),
13513 title: "t".into()
13514 })),
13515 PrPlan::Adopt {
13516 url: "u".into(),
13517 title: "t".into()
13518 }
13519 );
13520 let PrPlan::Stop(many) =
13521 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13522 else {
13523 panic!("many must stop");
13524 };
13525 assert!(many.contains('a') && many.contains('b'));
13526 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13527 panic!("a failed lookup must stop");
13528 };
13529 assert!(err.contains("bad token"));
13530 }
13531
13532 use super::*;
13533 use crate::run::GateStatus;
13534 use std::collections::BTreeMap;
13535 use std::time::Duration;
13536
13537 fn conductor() -> AgentSpec {
13538 AgentSpec {
13539 id: "conductor".to_owned(),
13540 kind: crate::config::AgentKind::Command,
13541 model: None,
13542 command: vec!["true".to_owned()],
13543 extra_args: Vec::new(),
13544 env: BTreeMap::new(),
13545 prompt_delivery: None,
13546 max_turn_secs: None,
13547 }
13548 }
13549
13550 fn spec(id: &str) -> AgentSpec {
13551 AgentSpec {
13552 id: id.to_owned(),
13553 kind: crate::config::AgentKind::Command,
13554 model: None,
13555 command: vec!["true".to_owned()],
13556 extra_args: Vec::new(),
13557 env: BTreeMap::new(),
13558 prompt_delivery: None,
13559 max_turn_secs: None,
13560 }
13561 }
13562
13563 fn ids(xs: &[&str]) -> BTreeSet<String> {
13564 xs.iter().map(|s| (*s).to_owned()).collect()
13565 }
13566
13567 #[test]
13568 fn pick_successor_skips_an_agent_another_seat_holds() {
13569 let roster = [spec("a"), spec("b"), spec("c")];
13571 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13572 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13573 }
13574
13575 #[test]
13576 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13577 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13579 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13580 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13581 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13582 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13583 }
13584
13585 #[test]
13586 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13587 let roster = [spec("a"), spec("b")];
13588 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13589 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13590 let carried = ids(&["b"]);
13591 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13592 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13593 }
13594
13595 #[test]
13596 fn pick_successor_returns_none_without_any_untried_successor() {
13597 let roster = [spec("a"), spec("b")];
13598 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13599 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13600 let carried = ids(&["a"]);
13601 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13602 }
13603
13604 #[test]
13605 fn pick_successor_rescue_avoids_another_seats_occupant() {
13606 let roster = [spec("a"), spec("b"), spec("c")];
13608 let carried = ids(&["b", "c"]);
13609 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13610 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13611 }
13612
13613 #[test]
13614 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13615 let roster = [spec("a"), spec("b"), spec("c")];
13616 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13617 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13618 }
13619
13620 #[test]
13621 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13622 let roster = [spec("a"), spec("b"), spec("c")];
13623 let failed = ids(&["a", "b", "c"]);
13624 let mut tried = ids(&["b"]);
13626 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13627 assert_eq!(first.id, "a");
13628 tried.insert(first.id.clone());
13629 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13630 assert_eq!(second.id, "c");
13631 tried.insert(second.id.clone());
13632 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13633 }
13634
13635 #[test]
13636 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13637 let roster = [spec("a"), spec("b")];
13638 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13639 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13640 }
13641
13642 #[test]
13643 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13644 let roster = [spec("a"), spec("b"), spec("c")];
13645 let h = SeatHistory {
13646 failed: ids(&["a"]),
13647 last_ok: Some("b".to_owned()),
13648 last_fail: None,
13649 };
13650 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13651 let gone = SeatHistory {
13653 last_ok: Some("zzz".to_owned()),
13654 ..h.clone()
13655 };
13656 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13657 let failed = SeatHistory {
13658 failed: ids(&["a", "b"]),
13659 last_ok: Some("b".to_owned()),
13660 last_fail: None,
13661 };
13662 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13663 }
13664
13665 #[test]
13666 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13667 let first = SeatState::new("review-1", "alpha", 7);
13668 let next = handover_seat("review-1", "gamma", 7);
13669 assert_ne!(first.claude_session, next.claude_session);
13670 }
13671
13672 #[test]
13673 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13674 let mut state = state_with_summary("x", "y");
13675 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13676 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13677 assert_ne!(a.claude_session, b.claude_session);
13678 }
13679
13680 #[test]
13681 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13682 let roster = [spec("a"), spec("b")];
13683 let h = SeatHistory {
13684 failed: ids(&["a", "b"]),
13685 ..SeatHistory::default()
13686 };
13687 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13688 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13689 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13690 }
13691
13692 #[test]
13699 fn failure_signature_ignores_numbers_and_paths() {
13700 assert_eq!(
13701 failure_signature("exited with Some(2) and no usable output"),
13702 failure_signature("exited with Some(137) and no usable output")
13703 );
13704 assert_eq!(
13705 failure_signature("process exited with no usable output (exit code 2)"),
13706 failure_signature("process exited with no usable output (exit code 137)")
13707 );
13708 for (old, new) in [
13709 (
13710 "exited with Some(2) and no usable output",
13711 "process exited with no usable output (exit code 2)",
13712 ),
13713 (
13714 "exited with None and no usable output",
13715 "process exited with no usable output (exit code unavailable)",
13716 ),
13717 ] {
13718 let previous = FailClass::Other(failure_signature(old));
13719 let current = FailClass::Other(failure_signature(new));
13720 assert_eq!(previous, current, "resumption retains the failure class");
13721 assert!(!should_hand_over(Some(&previous), ¤t));
13722 }
13723 assert_eq!(
13724 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13725 failure_signature("cannot open /var/x.txt: denied")
13726 );
13727 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13728 }
13729
13730 #[test]
13731 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13732 let other = FailClass::Other("x".into());
13733 assert!(should_hand_over(None, &FailClass::Quota));
13734 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13735 assert!(should_hand_over(
13736 Some(&FailClass::Timeout),
13737 &FailClass::Timeout
13738 ));
13739 assert!(should_hand_over(None, &other));
13740 assert!(!should_hand_over(Some(&other), &other));
13741 assert!(should_hand_over(
13742 Some(&other),
13743 &FailClass::Other("y".into())
13744 ));
13745 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13747 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13748 }
13749
13750 #[test]
13751 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13752 let a = SeatState::new("judge-1", "alpha", 7);
13753 let b = handover_seat("judge-1", "beta", 7);
13754 assert_ne!(a.claude_session, b.claude_session);
13755 assert_eq!(b.turns, 0);
13756 }
13757
13758 #[test]
13759 fn a_timeout_is_classified_apart_from_other_failures() {
13760 assert_eq!(
13761 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13762 Some(FailClass::Timeout)
13763 );
13764 assert!(matches!(
13765 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13766 Some(FailClass::Other(_))
13767 ));
13768 }
13769
13770 #[test]
13771 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13772 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13773 let tried = BTreeSet::from(["beta".to_owned()]);
13774 let next = next_untried_in_roster(&roster, 1, &tried);
13777 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13778 }
13779
13780 #[test]
13781 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13782 let roster = vec![spec("alpha"), spec("beta")];
13783 let tried = BTreeSet::from(["beta".to_owned()]);
13784 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13788 }
13789
13790 #[test]
13791 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13792 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13793 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13794 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13798 }
13799
13800 #[test]
13801 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13802 let roster = vec![spec("a"), spec("a"), spec("b")];
13803 let tried = BTreeSet::from(["a".to_owned()]);
13804 let next = next_untried_in_roster(&roster, 0, &tried);
13805 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13806 }
13807
13808 #[test]
13809 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13810 let roster = vec![spec("a"), spec("b")];
13811 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13812 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13813 }
13814
13815 #[test]
13816 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13817 let dir = tempfile::tempdir().unwrap();
13818 let bay = dir.path().join("ffff");
13819
13820 remove_if_empty(&bay);
13822 assert!(!bay.exists());
13823
13824 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13827 remove_if_empty(&bay);
13828 assert!(bay.exists(), "non-empty directory must survive");
13829
13830 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13832 remove_if_empty(&bay);
13833 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13834 }
13835
13836 #[test]
13845 fn a_full_panel_that_found_nothing_is_clean() {
13846 assert!(round_is_clean(
13847 0,
13848 true,
13849 2,
13850 2,
13851 0,
13852 IncompleteReviewPolicy::Block
13853 ));
13854 }
13855
13856 #[test]
13857 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13858 assert!(!round_is_clean(
13859 0,
13860 true,
13861 1,
13862 2,
13863 0,
13864 IncompleteReviewPolicy::Block
13865 ));
13866 }
13867
13868 #[test]
13869 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13870 assert!(!round_is_clean(
13871 1,
13872 true,
13873 1,
13874 2,
13875 0,
13876 IncompleteReviewPolicy::Warn
13877 ));
13878 }
13879
13880 #[test]
13881 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13882 assert!(round_is_clean(
13883 0,
13884 true,
13885 1,
13886 2,
13887 0,
13888 IncompleteReviewPolicy::Warn
13889 ));
13890 }
13891
13892 #[test]
13893 fn a_full_panel_with_an_open_finding_is_not_clean() {
13894 assert!(!round_is_clean(
13895 1,
13896 true,
13897 2,
13898 2,
13899 0,
13900 IncompleteReviewPolicy::Block
13901 ));
13902 }
13903
13904 #[test]
13905 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13906 assert!(!round_is_clean(
13907 0,
13908 false,
13909 2,
13910 2,
13911 0,
13912 IncompleteReviewPolicy::Block
13913 ));
13914 }
13915
13916 #[test]
13923 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13924 assert!(round_is_clean(
13927 0,
13928 true,
13929 1,
13930 2,
13931 1,
13932 IncompleteReviewPolicy::Block
13933 ));
13934 }
13935
13936 #[test]
13937 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13938 assert!(!round_is_clean(
13941 0,
13942 true,
13943 1,
13944 2,
13945 0,
13946 IncompleteReviewPolicy::Block
13947 ));
13948 }
13949
13950 #[test]
13951 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13952 assert!(!round_is_clean(
13953 1,
13954 true,
13955 1,
13956 2,
13957 1,
13958 IncompleteReviewPolicy::Block
13959 ));
13960 assert!(!round_is_clean(
13961 0,
13962 false,
13963 1,
13964 2,
13965 1,
13966 IncompleteReviewPolicy::Block
13967 ));
13968 }
13969
13970 #[test]
13971 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13972 assert!(!round_is_clean(
13976 0,
13977 true,
13978 0,
13979 2,
13980 2,
13981 IncompleteReviewPolicy::Block
13982 ));
13983 }
13984
13985 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13986 CommandOutcome {
13987 command: "test".to_owned(),
13988 code,
13989 output_tail: String::new(),
13990 duration_ms: 0,
13991 resource_blocked,
13992 }
13993 }
13994
13995 #[test]
13996 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13997 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13998 assert!(
13999 !verify_inconclusive(&[outcome(Some(1), false)]),
14000 "an ordinary failure is still evidence about the patch"
14001 );
14002 assert!(verify_inconclusive(&[outcome(None, true)]));
14003 assert!(
14004 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14005 "one inconclusive outcome taints the whole batch"
14006 );
14007 assert!(!verify_inconclusive(&[]));
14008 }
14009
14010 #[tokio::test]
14011 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14012 let calls = std::sync::atomic::AtomicUsize::new(0);
14016 let started = Instant::now();
14017 wait_for_pids_with(
14018 &[123],
14019 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14020 Duration::from_millis(5),
14021 Duration::from_secs(5),
14022 )
14023 .await;
14024 assert!(
14025 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14026 "must keep checking rather than deciding on the first answer"
14027 );
14028 assert!(
14029 started.elapsed() < Duration::from_secs(1),
14030 "must return the moment it is confirmed dead, not wait out the ceiling"
14031 );
14032 }
14033
14034 #[tokio::test]
14035 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14036 let started = Instant::now();
14037 wait_for_pids_with(
14038 &[123],
14039 |_| true, Duration::from_millis(5),
14041 Duration::from_millis(30),
14042 )
14043 .await;
14044 let elapsed = started.elapsed();
14045 assert!(
14046 elapsed >= Duration::from_millis(30),
14047 "must not give up before its own ceiling: {elapsed:?}"
14048 );
14049 assert!(
14050 elapsed < Duration::from_secs(1),
14051 "must not wait past its own ceiling either: {elapsed:?}"
14052 );
14053 }
14054
14055 #[tokio::test]
14056 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14057 let started = Instant::now();
14058 wait_for_pids_with(
14059 &[],
14060 |_| true,
14061 Duration::from_secs(5),
14062 Duration::from_secs(5),
14063 )
14064 .await;
14065 assert!(
14066 started.elapsed() < Duration::from_millis(200),
14067 "an empty pid list has nothing to confirm"
14068 );
14069 }
14070
14071 fn review_round(
14077 clean: bool,
14078 blocking: usize,
14079 answered: usize,
14080 expected: usize,
14081 progressed: bool,
14082 e2e_ok: bool,
14083 ) -> ReviewRound {
14084 ReviewRound {
14085 round: 1,
14086 head: "h".to_owned(),
14087 verified_head: None,
14088 verified_at: None,
14089 reviews: Vec::new(),
14090 e2e: vec![CommandOutcome {
14091 command: "test".to_owned(),
14092 code: Some(if e2e_ok { 0 } else { 1 }),
14093 output_tail: String::new(),
14094 duration_ms: 0,
14095 resource_blocked: false,
14096 }],
14097 verify_retried: false,
14098 e2e_deferred: false,
14099 e2e_defer_reason: None,
14100 fix: None,
14101 blocking,
14102 answered,
14103 expected,
14104 clean,
14105 progressed,
14106 vote_split: false,
14107 reconsideration: Vec::new(),
14108 verdict: None,
14109 }
14110 }
14111
14112 #[test]
14113 fn review_conclusion_is_none_when_nothing_has_run() {
14114 assert_eq!(review_conclusion(&[], 3), None);
14115 }
14116
14117 #[test]
14118 fn review_conclusion_is_none_while_rounds_remain() {
14119 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14120 assert_eq!(review_conclusion(&rounds, 3), None);
14121 }
14122
14123 #[test]
14124 fn review_conclusion_is_gating_once_a_round_is_clean() {
14125 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14126 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14127 }
14128
14129 #[test]
14130 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14131 let rounds = vec![
14132 review_round(false, 1, 2, 2, true, true),
14133 review_round(false, 1, 2, 2, true, true),
14134 ];
14135 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14136 }
14137
14138 #[test]
14139 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14140 let rounds = vec![
14141 review_round(false, 1, 2, 2, true, true),
14142 review_round(false, 1, 2, 2, true, false),
14143 ];
14144 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14145 }
14146
14147 #[test]
14148 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14149 let mut blocked = review_round(false, 1, 2, 2, true, false);
14156 blocked.e2e[0].resource_blocked = true;
14157 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14158 assert_eq!(review_conclusion(&rounds, 2), None);
14159 }
14160
14161 #[test]
14162 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14163 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14165 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14166 }
14167
14168 #[test]
14169 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14170 let rounds = vec![
14171 review_round(false, 1, 2, 2, false, true),
14172 review_round(false, 1, 2, 2, false, true),
14173 ];
14174 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14175 }
14176
14177 fn secs(n: u64) -> Duration {
14178 Duration::from_secs(n)
14179 }
14180
14181 #[tokio::test]
14184 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14185 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14186 if std::env::var_os(CHILD).is_none() {
14187 let output = std::process::Command::new(std::env::current_exe().unwrap())
14191 .args([
14192 "--exact",
14193 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14194 "--nocapture",
14195 ])
14196 .env(CHILD, "1")
14197 .env_remove("MAGI_WORKER_HOME")
14198 .env_remove("MAGI_WORKER_ID")
14199 .output()
14200 .unwrap();
14201 assert!(
14202 output.status.success()
14203 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14204 "{}\n{}",
14205 String::from_utf8_lossy(&output.stdout),
14206 String::from_utf8_lossy(&output.stderr)
14207 );
14208 return;
14209 }
14210 ask_test_home();
14211 let tmp = tempfile::tempdir().unwrap();
14212 let repo = tmp.path().join("repo");
14213 std::fs::create_dir_all(&repo).unwrap();
14214 init_repo(&repo);
14215 let g = |args: &[&str]| {
14216 let out = std::process::Command::new("git")
14217 .args(args)
14218 .current_dir(&repo)
14219 .quiet()
14220 .output()
14221 .unwrap();
14222 assert!(
14223 out.status.success(),
14224 "{}",
14225 String::from_utf8_lossy(&out.stderr)
14226 );
14227 };
14228 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14229 g(&["checkout", "-b", "historical"]);
14230 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14231 g(&["add", "."]);
14232 g(&["commit", "-m", "old alternative"]);
14233 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14234 g(&["checkout", "-b", "magi/task-selected", &base]);
14235 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14236 g(&["add", "."]);
14237 g(&["commit", "-m", "measurement infrastructure"]);
14238 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14239 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14240 g(&["add", "."]);
14241 g(&["commit", "-m", "dependent implementation"]);
14242 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14243 g(&["checkout", "main"]);
14244 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14245 g(&["add", "."]);
14246 g(&["commit", "-m", "unrelated base change"]);
14247 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14248 let origin = tmp.path().join("origin.git");
14249 g(&[
14250 "clone",
14251 "--bare",
14252 repo.to_str().unwrap(),
14253 origin.to_str().unwrap(),
14254 ]);
14255 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14256 g(&["fetch", "origin"]);
14257 let selection = crate::pr_recovery::Selection {
14258 url: "https://example.test/project/pull/1".into(),
14259 branch: "magi/task-selected".into(),
14260 head: head.clone(),
14261 source_head: None,
14262 source_run: None,
14263 source_proof: None,
14264 disposition: Some(crate::pr_recovery::Disposition::Review),
14265 instruction: "preserve measured behavior".into(),
14266 approved_at: jiff::Timestamp::now(),
14267 };
14268 let mut template = runner_at(RunStatus::Reviewing);
14269 template.state.repo = repo.clone();
14270 template.state.base_commit = current.clone();
14271 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14272 template.state.origin = Some(crate::run::Origin::queue("selected"));
14273 template.state.pr_head = Some(selection.branch.clone());
14274 template.state.pr_recovery = Some(selection.clone());
14275 let mut runner =
14276 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14277 .await
14278 .unwrap();
14279 let wt = runner.state.winner().unwrap().worktree.clone();
14280 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14281 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14282 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14283 assert!(runner.state.seeds.is_empty());
14284 assert_eq!(runner.state.instruction, "preserve measured behavior");
14285 runner.sync_to_base().await.unwrap();
14286 assert_eq!(
14287 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14288 "prerequisite"
14289 );
14290 assert_eq!(
14291 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14292 "measured implementation"
14293 );
14294 assert_eq!(
14295 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14296 "current main change"
14297 );
14298 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14299 assert_eq!(
14300 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14301 head
14302 );
14303
14304 let mut partial = runner_at(RunStatus::Reviewing);
14306 partial.state.repo = repo.clone();
14307 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14308 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14309 run: Some("exit 1".into()),
14310 ..Default::default()
14311 }];
14312 let selection = runner.state.pr_recovery.clone().unwrap();
14313 partial.state.pr_recovery = Some(selection.clone());
14314 partial.state.pr_head = Some(selection.branch.clone());
14315 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14316 let id = partial.state.id.clone();
14317 let error = match Runner::open_selected(
14318 &repo,
14319 partial.state,
14320 partial.roles,
14321 1,
14322 runner.state.base_commit.clone(),
14323 selection,
14324 )
14325 .await
14326 {
14327 Ok(_) => panic!("setup must fail"),
14328 Err(e) => e,
14329 };
14330 assert_eq!(
14331 error
14332 .downcast_ref::<crate::run::StartupFailure>()
14333 .unwrap()
14334 .run,
14335 id
14336 );
14337 let failed = RunState::load(&id).unwrap();
14338 assert_eq!(failed.status, RunStatus::Failed);
14339 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14340 assert!(Runner::resume(&id).is_err());
14341 runner.state.status = RunStatus::Ready;
14342 runner.state.driver_exited = true;
14343 runner.state.save().unwrap();
14344 let mut retry = runner_at(RunStatus::Reviewing);
14345 retry.state.repo = repo.clone();
14346 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14347 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14348 reselected.approved_at = jiff::Timestamp::now();
14349 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14350 retry.state.pr_head = Some(reselected.branch.clone());
14351 retry.state.pr_recovery = Some(reselected.clone());
14352 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14353 .await
14354 .unwrap();
14355 let restarted = Runner::open_selected(
14356 &repo,
14357 retry.state,
14358 retry.roles,
14359 1,
14360 runner.state.base_commit.clone(),
14361 reselected,
14362 )
14363 .await
14364 .unwrap();
14365 assert_eq!(
14366 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14367 .await
14368 .unwrap(),
14369 head
14370 );
14371 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14372 assert_eq!(
14373 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14374 head
14375 );
14376 }
14377
14378 #[tokio::test]
14379 #[cfg(unix)]
14380 async fn fallback_and_interrupted_restoration_require_coordination() {
14381 use std::os::fd::AsRawFd as _;
14382 ask_test_home();
14383 let tmp = tempfile::tempdir().unwrap();
14384 let mut runner = registered_runner(tmp.path()).await;
14385 runner.state.config.graph.timeout_resource_wait = 1;
14386 let winner = runner.state.candidates[0].clone();
14387 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14388 let leftover = winner.worktree.join("leftover");
14389 std::fs::write(&leftover, "untrusted edits").unwrap();
14390 let next = tmp.path().join("next-fixer");
14391 let mut failed = spec("first");
14392 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14393 let mut successor = spec("second");
14394 successor.command = vec![
14395 "sh".into(),
14396 "-c".into(),
14397 format!("touch '{}'; echo done", next.display()),
14398 ];
14399 let attempts = vec![
14400 (failed.clone(), "first".into()),
14401 (successor.clone(), "second".into()),
14402 ];
14403 let lock_path = runner.state.repo.join(".git/updater.lock");
14404 let external = std::fs::File::create(&lock_path).unwrap();
14405 assert_eq!(
14406 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14407 0
14408 );
14409 let artifacts = tmp.path().join("artifacts");
14410 let build = |spec, seat| SeatJob {
14411 spec,
14412 seat,
14413 cwd: winner.worktree.clone(),
14414 prompt: "fixture".into(),
14415 timeout: Duration::from_secs(10),
14416 allow_write: true,
14417 sessions: false,
14418 artifacts: artifacts.clone(),
14419 stem: "fix".into(),
14420 handover: None,
14421 };
14422 let (_, _, out) = runner
14423 .ask_fixer_among(
14424 attempts.clone(),
14425 "graft",
14426 None,
14427 Some((&winner.worktree, &head)),
14428 build,
14429 )
14430 .await;
14431 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14432 assert!(leftover.exists(), "contention must prevent clean");
14433 assert!(!next.exists());
14434 assert!(runner.state.handovers.is_empty());
14435 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14437 from_run: "earlier".into(),
14438 generation: 1,
14439 mode: crate::recovery::Mode::VerifyRepair,
14440 reasons: Vec::new(),
14441 reviewed_head: None,
14442 rounds: Vec::new(),
14443 open_findings: Vec::new(),
14444 evidence: None,
14445 criteria_inherited: true,
14446 repair: crate::recovery::Repair::Started {
14447 before: head.clone(),
14448 },
14449 });
14450 assert!(
14451 runner
14452 .recovery_repair(&winner)
14453 .await
14454 .unwrap_err()
14455 .is::<crate::repo_lock::ResourceBlocked>()
14456 );
14457 assert!(matches!(
14458 runner.state.recovery.as_ref().unwrap().repair,
14459 crate::recovery::Repair::Started { .. }
14460 ));
14461 assert!(leftover.exists());
14462 drop(external);
14463 runner.recovery_repair(&winner).await.unwrap();
14464 assert!(!leftover.exists());
14465 assert!(matches!(
14466 runner.state.recovery.as_ref().unwrap().repair,
14467 crate::recovery::Repair::Done { .. }
14468 ));
14469 std::fs::remove_file(&lock_path).unwrap();
14471 std::fs::create_dir(&lock_path).unwrap();
14472 std::fs::write(&leftover, "untrusted edits").unwrap();
14473 let (_, _, out) = runner
14474 .ask_fixer_among(
14475 attempts,
14476 "graft",
14477 None,
14478 Some((&winner.worktree, &head)),
14479 build,
14480 )
14481 .await;
14482 assert!(matches!(out, AgentOutcome::Failed(_)));
14483 assert!(leftover.exists());
14484 assert!(!next.exists());
14485 assert_eq!(
14486 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14487 head
14488 );
14489 }
14490
14491 #[cfg(unix)]
14492 async fn registered_runner(root: &Path) -> Runner {
14493 repository_runner(root, crate::config::test_coordination()).await
14494 }
14495
14496 async fn repository_runner(
14497 root: &Path,
14498 coordination: Option<crate::config::Coordination>,
14499 ) -> Runner {
14500 ask_test_home();
14501 let repo = root.join("repo");
14502 std::fs::create_dir_all(&repo).unwrap();
14503 init_repo(&repo);
14504 if coordination.is_some() {
14505 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14506 }
14507 let mut runner = runner_at(RunStatus::Prep);
14508 runner.state.repo = repo.clone();
14509 runner.state.config.coordination = coordination;
14510 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14511 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14512 runner.state.config.graph.advise = false;
14513 runner.roles.implementers = vec![spec("implement")];
14514 runner.roles.judges = vec![spec("judge")];
14515 runner.prep().await.unwrap();
14516 runner
14517 }
14518
14519 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14520
14521 #[tokio::test]
14522 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14523 let tmp = tempfile::tempdir().unwrap();
14524 let mut runner = repository_runner(tmp.path(), None).await;
14525 let prompts = runner.state.config.prompts.clone();
14526 let run = runner.state.id.clone();
14527 let ctx = WaveCtx {
14528 carry_seats: false,
14529 pauses: None,
14530 run: &run,
14531 node: "advise",
14532 prompts: &prompts,
14533 cache: None,
14534 round: None,
14535 };
14536 let jobs: Vec<_> = ["alpha", "beta"]
14537 .into_iter()
14538 .map(|id| SeatJob {
14539 spec: scripted(id, AUTH_OUTAGE),
14540 seat: SeatState::new(id, id, 7),
14541 prompt: "propose".into(),
14542 cwd: runner.state.candidates[0].worktree.clone(),
14543 timeout: Duration::from_secs(10),
14544 allow_write: false,
14545 sessions: false,
14546 artifacts: agent::artifacts_dir(&runner.state.dir()),
14547 stem: id.into(),
14548 handover: None,
14549 })
14550 .collect();
14551 let mut losses = Vec::new();
14552 let results = ask_wave_with(
14553 jobs.clone(),
14554 runner.sem.clone(),
14555 2,
14556 &[],
14557 &ctx,
14558 &mut losses,
14559 &mut runner.state,
14560 &|_: &str| Ok(()),
14561 )
14562 .await;
14563 assert!(
14564 results
14565 .iter()
14566 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14567 );
14568 assert_eq!(runner.state.quota.len(), 2);
14569 assert!(
14570 runner
14571 .state
14572 .quota
14573 .iter()
14574 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14575 );
14576 assert!(!runner.state.advise_attempted);
14577 runner.stop_for_resource().unwrap();
14578 assert_eq!(runner.state.status, RunStatus::Blocked);
14579 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14580 assert_eq!(
14581 runner.state.resource_block.as_ref().unwrap().kind,
14582 "agent authentication"
14583 );
14584 runner.state.resource_block = None;
14587 runner.state.quota.clear();
14588 let mut mixed = jobs;
14589 mixed[1].spec.command = vec!["false".into()];
14590 let results = ask_wave_with(
14591 mixed,
14592 runner.sem.clone(),
14593 0,
14594 &[],
14595 &ctx,
14596 &mut losses,
14597 &mut runner.state,
14598 &|_: &str| Ok(()),
14599 )
14600 .await;
14601 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14602 assert!(runner.state.resource_block.is_none());
14603 assert_eq!(losses.len(), 1);
14604 }
14605
14606 #[tokio::test]
14607 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14608 let tmp = tempfile::tempdir().unwrap();
14609 let mut runner = repository_runner(tmp.path(), None).await;
14610 let implementer = scripted("implement", AUTH_OUTAGE);
14611 runner.roles.implementers = vec![implementer.clone()];
14612 runner.state.config.agents = vec![implementer];
14613 runner.implement().await.unwrap();
14614 assert_eq!(runner.state.status, RunStatus::Blocked);
14615 assert_eq!(runner.state.quota.len(), 1);
14616 assert_eq!(
14617 runner.state.quota[0].cause,
14618 agent::QuotaCause::Authentication
14619 );
14620 let candidate = &runner.state.candidates[0];
14621 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14622 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14623 }
14624
14625 #[tokio::test]
14626 #[cfg(unix)]
14627 async fn parked_implementer_persists_completed_work() {
14628 for registered in [false, true] {
14629 let tmp = tempfile::tempdir().unwrap();
14630 let mut runner = registered_runner(tmp.path()).await;
14631 if !registered {
14632 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14633 }
14634 let marker = tmp.path().join("started");
14635 let release = tmp.path().join("release");
14636 let mut implementer = spec("implement");
14637 implementer.command = vec![
14638 "sh".into(),
14639 "-c".into(),
14640 format!(
14641 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14642 marker.display(),
14643 release.display()
14644 ),
14645 ];
14646 runner.roles.implementers = vec![implementer.clone()];
14647 runner.state.config.agents = vec![implementer];
14648 let pause = runner.pause.clone();
14649 let (result, ()) = tokio::join!(runner.implement(), async {
14650 tokio::time::timeout(Duration::from_secs(10), async {
14651 while !marker.exists() {
14652 tokio::time::sleep(Duration::from_millis(10)).await;
14653 }
14654 })
14655 .await
14656 .unwrap();
14657 pause.park();
14658 std::fs::write(&release, "go").unwrap();
14659 });
14660 result.unwrap();
14661 let candidate = &runner.state.candidates[0];
14662 assert!(candidate.commits > 0, "{candidate:?}");
14663 assert!(candidate.summary.contains("saved-result"));
14664 assert_eq!(candidate.files, 1);
14665 let before = candidate.commits;
14666 runner.implement().await.unwrap();
14667 assert_eq!(runner.state.candidates[0].commits, before);
14668 }
14669 }
14670
14671 #[tokio::test]
14672 #[cfg(unix)]
14673 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14674 let tmp = tempfile::tempdir().unwrap();
14675 let mut runner = registered_runner(tmp.path()).await;
14676 let winner = runner.state.candidates[0].clone();
14677 let cache = tmp.path().join("cache");
14678 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14679 runner.state.config.graph.timeout_resource_wait = 0;
14680 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14681 let crate::cache::AcquireOutcome::Acquired(held) =
14682 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14683 else {
14684 panic!("cache must be free")
14685 };
14686 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14687 from_run: "earlier".into(),
14688 generation: 1,
14689 mode: crate::recovery::Mode::VerifyRepair,
14690 reasons: Vec::new(),
14691 reviewed_head: None,
14692 rounds: Vec::new(),
14693 open_findings: Vec::new(),
14694 evidence: None,
14695 criteria_inherited: true,
14696 repair: crate::recovery::Repair::NotStarted,
14697 });
14698 runner.recovery_repair(&winner).await.unwrap();
14699 assert!(matches!(
14700 runner.state.recovery.as_ref().unwrap().repair,
14701 crate::recovery::Repair::WaitingForRestore { .. }
14702 ));
14703 drop(held);
14704 runner.state.resource_block = None;
14705 runner.state.config.graph.timeout_resource_wait = 1;
14706 runner.recovery_repair(&winner).await.unwrap();
14707 assert!(matches!(
14708 runner.state.recovery.as_ref().unwrap().repair,
14709 crate::recovery::Repair::Done { .. }
14710 ));
14711 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14712 }
14713
14714 #[tokio::test]
14715 #[cfg(unix)]
14716 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14717 use std::os::fd::AsRawFd as _;
14718 for graft in [false, true] {
14719 let tmp = tempfile::tempdir().unwrap();
14720 let mut runner = registered_runner(tmp.path()).await;
14721 runner.state.config.graph.timeout_resource_wait = 1;
14722 let winner = runner.state.candidates[0].clone();
14723 let marker = tmp.path().join("fixer-started");
14724 let mut fixer = spec("fixer");
14725 fixer.command = vec![
14726 "sh".into(),
14727 "-c".into(),
14728 format!("echo started >> '{}'; echo done", marker.display()),
14729 ];
14730 runner.roles.fixer = Some(vec![fixer]);
14731 if graft {
14732 let mut loser = winner.clone();
14733 loser.label = 'B';
14734 loser.index = 1;
14735 runner.state.candidates.push(loser);
14736 runner.state.config.graph.graft_runner_up = true;
14737 runner.state.tally = Some(Tally {
14738 first_choice: BTreeMap::new(),
14739 borda: BTreeMap::new(),
14740 winner: winner.label,
14741 rankings: 1,
14742 unanimous_initial: true,
14743 deliberated: false,
14744 changed_votes: 0,
14745 unanimous_final: true,
14746 tie_break: None,
14747 judges: 1,
14748 present: 1,
14749 quorum: 1,
14750 met_quorum: true,
14751 uncontested: None,
14752 });
14753 runner.state.judgements.push(crate::run::Judgement {
14754 judge: 1,
14755 seat: "judge-1".into(),
14756 agent: "judge".into(),
14757 ranking: vec![winner.label, 'B'],
14758 reasons: BTreeMap::new(),
14759 confidence: None,
14760 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14761 order: vec![0, 1],
14762 failed: None,
14763 duration_ms: 0,
14764 });
14765 } else {
14766 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14767 from_run: "earlier".into(),
14768 generation: 1,
14769 mode: crate::recovery::Mode::VerifyRepair,
14770 reasons: Vec::new(),
14771 reviewed_head: None,
14772 rounds: Vec::new(),
14773 open_findings: Vec::new(),
14774 evidence: None,
14775 criteria_inherited: true,
14776 repair: crate::recovery::Repair::NotStarted,
14777 });
14778 }
14779 let reader = crate::repo_lock::SeatLease::for_checkout(
14780 &winner.worktree,
14781 false,
14782 crate::config::test_coordination().as_ref(),
14783 )
14784 .await
14785 .unwrap();
14786 let pause = runner.pause.clone();
14787 let (result, ()) = tokio::join!(
14788 async {
14789 if graft {
14790 runner.graft().await
14791 } else {
14792 runner.recovery_repair(&winner).await
14793 }
14794 },
14795 async {
14796 tokio::time::sleep(Duration::from_millis(100)).await;
14797 pause.park();
14798 }
14799 );
14800 result.unwrap();
14801 assert!(!marker.exists(), "a blocked seat must not spawn");
14802 runner.state = RunState::load(&runner.state.id).unwrap();
14804 if graft {
14805 assert_eq!(
14806 runner.state.graft.as_ref().unwrap().outcome,
14807 GraftOutcome::WaitingForRestore
14808 );
14809 } else {
14810 assert!(matches!(
14811 runner.state.recovery.as_ref().unwrap().repair,
14812 crate::recovery::Repair::WaitingForRestore { .. }
14813 ));
14814 }
14815 drop(reader);
14816 runner.pause = Pause::new();
14817 runner.state.resource_block = None;
14818 let external =
14820 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14821 assert_eq!(
14822 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14823 0
14824 );
14825 let leftover = winner.worktree.join("leftover");
14826 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14827 let result = if graft {
14828 runner.graft().await
14829 } else {
14830 runner.recovery_repair(&winner).await
14831 };
14832 assert!(
14833 result
14834 .unwrap_err()
14835 .is::<crate::repo_lock::ResourceBlocked>()
14836 );
14837 assert!(leftover.exists());
14838 assert!(!marker.exists());
14839 drop(external);
14840 runner.state.resource_block = None;
14841 let result = if graft {
14842 runner.graft().await
14843 } else {
14844 runner.recovery_repair(&winner).await
14845 };
14846 result.unwrap();
14847 assert!(!leftover.exists(), "restore must precede retry");
14848 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14849 if graft {
14850 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14851 } else {
14852 assert!(matches!(
14853 runner.state.recovery.as_ref().unwrap().repair,
14854 crate::recovery::Repair::Done { .. }
14855 ));
14856 }
14857 }
14858 }
14859
14860 #[tokio::test]
14861 #[cfg(unix)]
14862 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14863 use std::os::fd::AsRawFd as _;
14864 let tmp = tempfile::tempdir().unwrap();
14865 let mut runner = registered_runner(tmp.path()).await;
14866 runner.state.config.graph.timeout_resource_wait = 1;
14867 let winner = runner.state.candidates[0].clone();
14868 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14869 runner.state.tally = Some(Tally {
14870 first_choice: BTreeMap::new(),
14871 borda: BTreeMap::new(),
14872 winner: winner.label,
14873 rankings: 0,
14874 unanimous_initial: false,
14875 deliberated: false,
14876 changed_votes: 0,
14877 unanimous_final: false,
14878 tie_break: None,
14879 judges: 0,
14880 present: 0,
14881 quorum: 0,
14882 met_quorum: true,
14883 uncontested: Some("fixture".into()),
14884 });
14885 runner.state.graft = Some(GraftRecord {
14886 pre: pre.clone(),
14887 post: None,
14888 lines: None,
14889 limit: None,
14890 agent: None,
14891 outcome: GraftOutcome::Running,
14892 requested: Vec::new(),
14893 adoption: None,
14894 adoption_error: None,
14895 });
14896 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14897 git::commit_all(&winner.worktree, "unfinished graft")
14898 .await
14899 .unwrap();
14900 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14901 let leftover = winner.worktree.join("leftover");
14902 std::fs::write(&leftover, "untrusted edits").unwrap();
14903 let path = runner.state.repo.join(".git/updater.lock");
14904 let external = std::fs::File::create(&path).unwrap();
14905 assert_eq!(
14906 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14907 0
14908 );
14909 assert!(
14910 runner
14911 .graft()
14912 .await
14913 .unwrap_err()
14914 .is::<crate::repo_lock::ResourceBlocked>()
14915 );
14916 assert!(matches!(
14917 runner.state.graft.as_ref().unwrap().outcome,
14918 GraftOutcome::Running
14919 ));
14920 assert_eq!(
14921 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14922 unfinished
14923 );
14924 assert!(leftover.exists());
14925 drop(external);
14926 std::fs::remove_file(&path).unwrap();
14927 std::fs::create_dir(&path).unwrap();
14928 assert!(runner.graft().await.is_err());
14929 assert!(matches!(
14930 runner.state.graft.as_ref().unwrap().outcome,
14931 GraftOutcome::Running
14932 ));
14933 assert_eq!(
14934 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14935 unfinished
14936 );
14937 assert!(leftover.exists());
14938 std::fs::remove_dir(&path).unwrap();
14939 runner.graft().await.unwrap();
14940 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14941 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14942 assert!(!leftover.exists());
14943 }
14944
14945 #[tokio::test]
14946 #[cfg(unix)]
14947 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14948 use std::os::fd::AsRawFd as _;
14949 let tmp = tempfile::tempdir().unwrap();
14950 let mut runner = registered_runner(tmp.path()).await;
14951 runner.state.prep_complete = Some(false);
14952 runner.state.config.graph.timeout_resource_wait = 1;
14953 runner.state.save().unwrap();
14954 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14955 assert_eq!(
14956 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14957 0
14958 );
14959 runner.execute().await.unwrap();
14960 let disk = RunState::load(&runner.state.id).unwrap();
14961 assert!(disk.driver_exited);
14962 assert_eq!(disk.driver_pid, None);
14963 assert_eq!(runner.state.status, RunStatus::Blocked);
14964 assert_eq!(
14965 runner.state.resource_block.as_ref().unwrap().kind,
14966 "repository coordination"
14967 );
14968 assert!(runner.state.prep_failure.is_none());
14969 assert!(runner.state.seats.is_empty());
14970 runner.pause.park();
14971 tokio::time::timeout(Duration::from_secs(10), runner.execute())
14974 .await
14975 .unwrap()
14976 .unwrap();
14977 assert!(runner.state.parked);
14978 assert!(runner.state.prep_failure.is_none());
14979 let disk = RunState::load(&runner.state.id).unwrap();
14980 assert!(disk.driver_exited);
14981 assert_eq!(disk.driver_pid, None);
14982 }
14983
14984 #[tokio::test]
14985 #[cfg(unix)]
14986 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14987 use std::os::fd::AsRawFd as _;
14988 let tmp = tempfile::tempdir().unwrap();
14989 let mut runner = registered_runner(tmp.path()).await;
14990 runner.state.prep_complete = Some(false);
14991 runner.state.config.graph.timeout_resource_wait = 30;
14992 runner.state.save().unwrap();
14993 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14994 assert_eq!(
14995 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14996 0
14997 );
14998 assert!(
14999 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15000 .await
15001 .is_err()
15002 );
15003 let disk = RunState::load(&runner.state.id).unwrap();
15004 assert!(disk.driver_exited);
15005 assert_eq!(disk.driver_pid, None);
15006 assert!(disk.status.resumable());
15007 assert!(runner.state.driver_exited);
15008 assert_eq!(runner.state.driver_pid, None);
15009 runner.state.save().unwrap();
15010 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15011 }
15012
15013 #[tokio::test]
15014 #[cfg(unix)]
15015 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15016 let tmp = tempfile::tempdir().unwrap();
15017 let mut runner = registered_runner(tmp.path()).await;
15018 let winner = runner.state.candidates[0].clone();
15019 let cache = tmp.path().join("cache");
15020 let started = tmp.path().join("pre-gate-started");
15021 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15022 runner.state.config.verify.gate =
15023 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15024 let home = crate::run::home();
15025 let owner =
15026 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15027 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15028 crate::cache::AcquireOutcome::Acquired(g) => g,
15029 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15030 };
15031 let mut gate = Box::pin(runner.run_gate(&winner));
15032 assert!(
15033 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15034 .await
15035 .is_err()
15036 );
15037 assert!(!started.exists(), "pre-gate must wait for the cache too");
15038 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15042 let editor = crate::repo_lock::SeatLease::checkout_with(
15043 &winner.worktree,
15044 true,
15045 crate::config::test_coordination().as_ref(),
15046 &wait,
15047 )
15048 .await
15049 .unwrap();
15050 drop(editor);
15051 guard.release();
15052 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15053 .await
15054 .unwrap()
15055 .unwrap();
15056 drop(gate);
15057 assert!(outcomes.iter().all(CommandOutcome::ok));
15058 assert!(started.exists());
15059 }
15060
15061 #[tokio::test]
15062 #[cfg(unix)]
15063 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15064 use std::os::fd::AsRawFd as _;
15065
15066 ask_test_home();
15067 let tmp = tempfile::tempdir().unwrap();
15068 let repo = tmp.path().join("repo");
15069 std::fs::create_dir_all(&repo).unwrap();
15070 init_repo(&repo);
15071 let common = repo.join(".git");
15072 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15073 let mut runner = runner_at(RunStatus::Prep);
15074 runner.state.repo = repo.clone();
15075 runner.state.config.coordination = crate::config::test_coordination();
15076 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15077 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15078 runner.state.config.graph.advise = false;
15079 runner.roles.implementers = vec![spec("implement")];
15080 runner.roles.judges = vec![spec("judge")];
15081 runner.prep().await.unwrap();
15082 let winner = runner.state.candidates[0].clone();
15083 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15086 let completed = tmp.path().join("seat-completed");
15087 runner.roles.implementers[0].command = vec![
15088 "sh".into(),
15089 "-c".into(),
15090 format!("touch '{}'; echo completed", completed.display()),
15091 ];
15092 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15093 assert_eq!(
15094 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15095 0
15096 );
15097 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15098 let mut implement = Box::pin(runner.implement());
15099 tokio::select! {
15100 result = &mut implement => panic!("rescue must wait: {result:?}"),
15101 result = tokio::time::timeout(Duration::from_secs(10), async {
15102 while !completed.exists() {
15103 tokio::time::sleep(Duration::from_millis(20)).await;
15104 }
15105 }) => result.unwrap(),
15106 }
15107 assert!(
15108 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15109 .await
15110 .is_err()
15111 );
15112 assert_eq!(
15113 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15114 before
15115 );
15116 assert_eq!(
15117 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15118 0
15119 );
15120 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15121 .await
15122 .unwrap()
15123 .unwrap();
15124 drop(implement);
15125 assert_ne!(
15126 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15127 before
15128 );
15129 assert!(git::is_clean(&winner.worktree).await.unwrap());
15130
15131 let started = tmp.path().join("pre-gate-started");
15132 let proceed = tmp.path().join("pre-gate-proceed");
15133 let checked = tmp.path().join("gate-ran");
15134 runner.state.config.verify.pre_gate = vec![format!(
15135 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15136 started.display(),
15137 proceed.display()
15138 )];
15139 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15140 assert_eq!(
15141 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15142 0
15143 );
15144 let mut gate = Box::pin(runner.run_gate(&winner));
15145 assert!(
15146 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15147 .await
15148 .is_err()
15149 );
15150 assert!(!started.exists());
15151 assert!(!checked.exists());
15152 assert!(git::is_clean(&winner.worktree).await.unwrap());
15153 assert_eq!(
15154 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15155 0
15156 );
15157 tokio::select! {
15158 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15159 result = tokio::time::timeout(Duration::from_secs(10), async {
15160 while !started.exists() {
15161 tokio::time::sleep(Duration::from_millis(20)).await;
15162 }
15163 }) => result.unwrap(),
15164 }
15165 assert_eq!(
15166 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15167 -1,
15168 "mechanical edits must hold the real lock"
15169 );
15170 std::fs::write(proceed, "ready").unwrap();
15171 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15172 .await
15173 .unwrap()
15174 .unwrap();
15175 drop(gate);
15176 assert!(outcomes.iter().all(CommandOutcome::ok));
15177 assert!(checked.exists());
15178 assert!(runner.state.pre_gate_commit.is_some());
15179 assert!(git::is_clean(&winner.worktree).await.unwrap());
15180 assert_eq!(
15181 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15182 0,
15183 "gate completion must release coordination"
15184 );
15185 drop(outsider);
15186 std::fs::remove_file(common.join("updater.lock")).unwrap();
15187 std::fs::create_dir(common.join("updater.lock")).unwrap();
15188 std::fs::remove_file(&started).unwrap();
15189 std::fs::remove_file(&checked).unwrap();
15190 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15191 let error = runner.run_gate(&winner).await.unwrap_err();
15192 assert!(
15193 error
15194 .to_string()
15195 .contains("acquire mandatory repository lock")
15196 );
15197 assert!(
15198 !started.exists(),
15199 "failed acquisition must prevent pre-gate"
15200 );
15201 assert!(!checked.exists(), "failed acquisition must prevent gate");
15202 assert!(git::is_clean(&winner.worktree).await.unwrap());
15203 assert_eq!(
15204 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15205 head
15206 );
15207 }
15208
15209 #[tokio::test]
15210 #[cfg(unix)]
15211 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15212 use std::os::fd::AsRawFd as _;
15213 use std::time::Duration;
15214
15215 ask_test_home();
15216 let tmp = tempfile::tempdir().unwrap();
15217 let repo = tmp.path().join("repo");
15218 std::fs::create_dir_all(&repo).unwrap();
15219 init_repo(&repo);
15220 let common = repo.join(".git");
15221 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15222 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15223 assert_eq!(
15224 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15225 0
15226 );
15227 let mut runner = runner_at(RunStatus::Prep);
15228 runner.state.repo = repo.clone();
15229 runner.state.config.coordination = crate::config::test_coordination();
15230 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15231 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15232 runner.state.config.graph.advise = true;
15233 runner.state.config.graph.advisors = 1;
15234 runner.roles.implementers = vec![spec("implement")];
15235 runner.roles.judges = vec![spec("judge")];
15236 let root = runner.state.worktree_root();
15237 let started = tmp.path().join("setup-started");
15238 let proceed = tmp.path().join("setup-proceed");
15239 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15240 run: Some(format!(
15241 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15242 started.display(),
15243 proceed.display()
15244 )),
15245 ..Default::default()
15246 }];
15247 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15248 let config = git::git(&repo, &["config", "--local", "--list"])
15249 .await
15250 .unwrap();
15251 let mut prep = Box::pin(runner.prep());
15252 assert!(
15253 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15254 .await
15255 .is_err()
15256 );
15257 assert!(
15258 !root.exists(),
15259 "contention must prevent even checkout creation"
15260 );
15261 assert!(!started.exists(), "contention must prevent setup");
15262 assert_eq!(
15263 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15264 branches
15265 );
15266 assert_eq!(
15267 git::git(&repo, &["config", "--local", "--list"])
15268 .await
15269 .unwrap(),
15270 config
15271 );
15272 drop(lock);
15273 tokio::select! {
15274 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15275 result = tokio::time::timeout(Duration::from_secs(10), async {
15276 while !started.exists() {
15277 tokio::time::sleep(Duration::from_millis(20)).await;
15278 }
15279 }) => result.unwrap(),
15280 }
15281 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15282 assert_eq!(
15283 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15284 -1
15285 );
15286 assert!(
15287 tokio::time::timeout(
15288 Duration::from_millis(100),
15289 crate::repo_lock::SeatLease::for_checkout(
15290 &repo,
15291 true,
15292 crate::config::test_coordination().as_ref()
15293 )
15294 )
15295 .await
15296 .is_err(),
15297 "preparation must also exclude local editors"
15298 );
15299 std::fs::write(proceed, "ready").unwrap();
15300 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15301 .await
15302 .unwrap()
15303 .unwrap();
15304 drop(prep);
15305 assert_eq!(runner.state.prep_complete, Some(true));
15306 assert!(root.join("judge-1").exists());
15307 assert!(root.join("advisor-1").exists());
15308 assert_eq!(
15309 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15310 0
15311 );
15312 }
15313
15314 #[tokio::test]
15315 #[cfg(unix)]
15316 async fn preparation_lock_failure_prevents_repository_changes() {
15317 ask_test_home();
15318 let tmp = tempfile::tempdir().unwrap();
15319 let repo = tmp.path().join("repo");
15320 std::fs::create_dir_all(&repo).unwrap();
15321 init_repo(&repo);
15322 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15323 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15326 let mut runner = runner_at(RunStatus::Prep);
15327 runner.state.repo = repo.clone();
15328 runner.state.config.coordination = crate::config::test_coordination();
15329 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15330 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15331 runner.roles.implementers = vec![spec("implement")];
15332 let error = runner.prep().await.unwrap_err();
15333 assert!(
15334 error
15335 .to_string()
15336 .contains("acquire mandatory repository lock")
15337 );
15338 assert!(!runner.state.worktree_root().exists());
15339 assert!(runner.state.candidates.is_empty());
15340 assert!(!repo.join(".git/config.worktree").exists());
15341 assert_eq!(
15342 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15343 "* main"
15344 );
15345 }
15346
15347 #[tokio::test]
15348 async fn preparation_records_creation_intent_before_git_can_fail() {
15349 ask_test_home();
15350 let tmp = tempfile::tempdir().unwrap();
15351 let repo = tmp.path().join("repo");
15352 std::fs::create_dir_all(&repo).unwrap();
15353 init_repo(&repo);
15354 let mut runner = runner_at(RunStatus::Prep);
15355 runner.state.repo = repo.clone();
15356 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15357 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15358 runner.state.config.blind.commit_msg_hook = false;
15359 runner.roles.implementers = vec![spec("one")];
15360 let label = blind::assign_labels(1, runner.state.seed)[0];
15361 let branch = runner.state.branch_for(label);
15362 git::git(&repo, &["branch", &branch]).await.unwrap();
15365 assert!(runner.prep().await.is_err());
15366 let saved = RunState::load(&runner.state.id).unwrap();
15367 assert!(saved.candidates.is_empty());
15368 let event = saved
15369 .events
15370 .iter()
15371 .find(|e| e.node == "prep-intent")
15372 .unwrap();
15373 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15374 assert_eq!(intent["branch"], branch);
15375 assert_eq!(intent["start"], runner.state.base_commit);
15376 assert_eq!(
15377 intent["worktree"],
15378 runner
15379 .state
15380 .worktree_root()
15381 .join(format!("cand-{label}"))
15382 .to_string_lossy()
15383 .as_ref()
15384 );
15385 assert_eq!(intent["picks"], serde_json::json!([]));
15386 }
15387
15388 #[tokio::test]
15389 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15390 ask_test_home();
15391 let tmp = tempfile::tempdir().unwrap();
15392 let repo = tmp.path().join("repo");
15393 std::fs::create_dir_all(&repo).unwrap();
15394 init_repo(&repo);
15395 let mut runner = runner_at(RunStatus::Prep);
15396 runner.state.repo = repo.clone();
15397 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15398 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15399 runner.state.sources.pick_commits.push(refs::Input {
15400 token: "missing".into(),
15401 sha: "0".repeat(40),
15402 provenance: refs::Provenance::ExplicitCli,
15403 });
15404 assert!(runner.prep().await.is_err());
15405 assert!(runner.state.candidates.is_empty());
15406 assert!(!runner.state.worktree_root().exists());
15407 }
15408
15409 #[tokio::test]
15410 async fn prose_and_legacy_inference_never_import_into_candidates() {
15411 ask_test_home();
15412 let tmp = tempfile::tempdir().unwrap();
15413 let repo = tmp.path().join("repo");
15414 std::fs::create_dir_all(&repo).unwrap();
15415 init_repo(&repo);
15416 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15417 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15418 .await
15419 .unwrap();
15420 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15421 git::git(&repo, &["add", "."]).await.unwrap();
15422 git::git(&repo, &["commit", "-m", "historical implementation"])
15423 .await
15424 .unwrap();
15425 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15426 git::git(&repo, &["checkout", "main"]).await.unwrap();
15427 for instruction in [
15428 format!("Example: magi/history/A at {source}"),
15429 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15430 format!("Never import magi/history/A or {source}"),
15431 format!("Conversation answer: I mentioned {source} only as background"),
15432 ] {
15433 let mut runner = runner_at(RunStatus::Prep);
15434 runner.state.repo = repo.clone();
15435 runner.state.base_commit = base.clone();
15436 runner.state.instruction = instruction.clone();
15437 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15438 runner.state.config.blind.commit_msg_hook = false;
15439 runner.state.seeds = vec![refs::Seed {
15440 token: "magi/history/A".into(),
15441 sha: source.clone(),
15442 kind: refs::SeedKind::Unmerged,
15443 branch: true,
15444 detail: String::new(),
15445 }];
15446 runner.roles.implementers = vec![spec("one")];
15447 assert_eq!(seeded_instruction(&runner.state), instruction);
15448 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15450 legacy.as_object_mut().unwrap().remove("sources");
15451 runner.state = serde_json::from_value(legacy).unwrap();
15452 assert_eq!(runner.state.sources, refs::Sources::default());
15453 runner.state.save().unwrap();
15454 runner.state = RunState::load(&runner.state.id).unwrap();
15455 runner.prep().await.unwrap();
15456 for candidate in &runner.state.candidates {
15457 assert_eq!(
15458 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15459 base
15460 );
15461 assert!(!candidate.worktree.join("unwanted").exists());
15462 }
15463 }
15464 }
15465
15466 #[tokio::test]
15467 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15468 ask_test_home();
15469 let tmp = tempfile::tempdir().unwrap();
15470 let repo = tmp.path().join("repo");
15471 std::fs::create_dir_all(&repo).unwrap();
15472 init_repo(&repo);
15473 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15474 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15475 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15476 git::git(&repo, &["add", "."]).await.unwrap();
15477 git::git(&repo, &["commit", "-m", "required seed"])
15478 .await
15479 .unwrap();
15480 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15481 git::git(&repo, &["checkout", "main"]).await.unwrap();
15482 let mut runner = runner_at(RunStatus::Prep);
15483 runner.state.repo = repo.clone();
15484 runner.state.base_commit = base.clone();
15485 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15486 runner.state.config.graph.advise = false;
15487 runner.state.config.blind.commit_msg_hook = false;
15488 runner.state.sources =
15489 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15490 .await
15491 .unwrap();
15492 runner.roles.implementers = vec![spec("one"), spec("two")];
15493 runner.roles.judges = vec![spec("judge")];
15494 let label = blind::assign_labels(2, runner.state.seed)[0];
15495 let branch = runner.state.branch_for(label);
15496 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15497 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15498 .await
15499 .unwrap();
15500 runner.state.candidates.push(Candidate {
15501 index: 0,
15502 label,
15503 agent: "one".into(),
15504 branch,
15505 worktree,
15506 summary: String::new(),
15507 stat: String::new(),
15508 files: 0,
15509 commits: 0,
15510 empty: false,
15511 failed: None,
15512 verified_noop: None,
15513 duration_ms: 0,
15514 folded: false,
15515 });
15516 runner.state.save().unwrap();
15517 runner.state = RunState::load(&runner.state.id).unwrap();
15519 assert_eq!(runner.state.prep_complete, Some(false));
15520 runner.prep().await.unwrap();
15521 assert_eq!(runner.state.prep_complete, Some(true));
15522 assert_eq!(runner.state.candidates.len(), 2);
15523 for candidate in &runner.state.candidates {
15524 assert_eq!(
15525 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15526 git::tree_of(&repo, &seed).await.unwrap()
15527 );
15528 assert_eq!(
15529 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15530 "required commit"
15531 );
15532 }
15533 assert!(runner.state.worktree_root().join("judge-1").exists());
15534 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15535 runner.prep().await.unwrap();
15536 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15537
15538 runner.state.prep_complete = Some(false);
15539 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15540 run: Some(format!(
15541 "test -f '{}'",
15542 tmp.path().join("dependency-ready").display()
15543 )),
15544 ..Default::default()
15545 }];
15546 assert!(runner.prep().await.is_err());
15547 assert!(runner.state.prep_failure.is_some());
15548 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15549 runner.state = RunState::load(&runner.state.id).unwrap();
15550 assert!(
15551 runner.prep().await.is_err(),
15552 "correcting the cause alone must not auto-retry"
15553 );
15554 let failed_at = runner.state.events.last().unwrap().at;
15555 assert!(
15556 runner.retry_preparation(failed_at).is_err(),
15557 "old approvals cannot unlock the hold"
15558 );
15559 let mut task = crate::queue::Task::new(
15560 "retry".into(),
15561 "retry setup".into(),
15562 repo.clone(),
15563 crate::queue::Source::Human,
15564 );
15565 task.start(runner.state.id.clone());
15566 task.hold_machine(Some("setup failed".into()));
15567 task.release_operator("cli", None);
15568 runner
15569 .retry_preparation(task.resumption.as_ref().unwrap().at)
15570 .unwrap();
15571 runner.prep().await.unwrap();
15572 assert!(runner.state.prep_failure.is_none());
15573 assert_eq!(runner.state.prep_complete, Some(true));
15574 assert_eq!(runner.state.candidates.len(), 2);
15575 assert!(runner.state.worktree_root().join("judge-1").exists());
15576 for candidate in &runner.state.candidates {
15577 assert_eq!(
15578 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15579 git::tree_of(&repo, &seed).await.unwrap()
15580 );
15581 }
15582 }
15583
15584 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15585 let mut heads = Vec::new();
15586 for candidate in candidates {
15587 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15588 }
15589 heads
15590 }
15591
15592 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15593 ask_test_home();
15594 let tmp = tempfile::tempdir().unwrap();
15595 let repo = tmp.path().join("repo");
15596 std::fs::create_dir(&repo).unwrap();
15597 init_repo(&repo);
15598 let mut runner = runner_at(RunStatus::Reviewing);
15599 runner.state.repo = repo.clone();
15600 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15601 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15602 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15603 let branch = format!("candidate-{label}");
15604 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15605 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15606 .await
15607 .unwrap();
15608 if label == 'B' {
15609 std::fs::write(worktree.join("change"), "winner").unwrap();
15610 git::rescue_commit(&worktree, "candidate change")
15611 .await
15612 .unwrap();
15613 }
15614 runner.state.candidates.push(Candidate {
15615 index,
15616 label,
15617 agent: "alpha".into(),
15618 branch,
15619 worktree,
15620 summary: "completed reply".into(),
15621 stat: String::new(),
15622 files: usize::from(label == 'B'),
15623 commits: usize::from(label == 'B'),
15624 empty: label == 'A',
15625 failed: None,
15626 verified_noop: None,
15627 duration_ms: 1,
15628 folded: false,
15629 });
15630 }
15631 runner.state.judge_skipped = true;
15632 runner.state.tally = Some(Tally {
15633 first_choice: BTreeMap::new(),
15634 borda: BTreeMap::new(),
15635 winner: 'B',
15636 rankings: 0,
15637 unanimous_initial: false,
15638 deliberated: false,
15639 changed_votes: 0,
15640 unanimous_final: false,
15641 tie_break: None,
15642 judges: 0,
15643 present: 0,
15644 quorum: 0,
15645 met_quorum: true,
15646 uncontested: Some("only B produced a change".into()),
15647 });
15648 runner.state.save().unwrap();
15649 (tmp, runner)
15650 }
15651
15652 #[tokio::test]
15653 #[cfg(unix)]
15654 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15655 for (solo, failed_checkpoint) in
15656 [(true, false), (false, false), (true, true), (false, true)]
15657 {
15658 let (_tmp, mut runner) = preservation_resume_fixture().await;
15659 if solo {
15660 runner.state.candidates.retain(|c| c.label == 'A');
15661 }
15662 runner.state.candidates[0].empty = false;
15663 runner.state.candidates[0].summary.clear();
15664 runner.state.tally = None;
15665 runner.state.judge_skipped = false;
15666 runner.state.status = RunStatus::Implementing;
15667 runner.state.config.disk.min_free_bytes = u64::MAX;
15668 let mut implementer = spec("alpha");
15669 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15670 runner.roles.implementers = vec![implementer];
15671 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15672 if failed_checkpoint {
15673 std::fs::create_dir_all(&patch).unwrap();
15674 }
15675 runner.state.save().unwrap();
15676 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15677 runner.state = RunState::load(&runner.state.id).unwrap();
15678 assert_eq!(runner.state.status, RunStatus::Blocked);
15679 assert!(runner.state.interrupted_implementations.contains(&'A'));
15680 assert_eq!(runner.state.candidates[0].commits, 0);
15681 assert!(!runner.state.candidates[0].empty);
15682 assert!(runner.state.candidates[0].failed.is_none());
15683 if failed_checkpoint {
15684 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15685 std::fs::remove_dir(patch).unwrap();
15686 }
15687 runner.state.resource_block = None;
15688 runner.state.config.disk.min_free_bytes = 0;
15689 runner.roles.implementers[0].command = vec![
15690 "sh".into(),
15691 "-c".into(),
15692 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15693 ];
15694 runner.implement().await.unwrap();
15695 assert!(runner.state.interrupted_implementations.is_empty());
15696 assert!(runner.state.candidate_preservation.is_empty());
15697 assert_eq!(runner.state.candidates[0].commits, 1);
15698 assert!(!runner.state.candidates[0].empty);
15699 assert!(
15700 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15701 .unwrap()
15702 .contains("retried empty interrupted seat")
15703 );
15704 if !solo {
15705 assert_eq!(runner.state.candidates[1].commits, 1);
15706 }
15707 }
15708 }
15709
15710 #[tokio::test]
15711 #[cfg(unix)]
15712 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15713 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15714 let (_tmp, mut runner) = preservation_resume_fixture().await;
15715 runner.state.candidates.retain(|c| c.label == 'A');
15716 runner.state.candidates[0].empty = false;
15717 runner.state.candidates[0].summary.clear();
15718 runner.state.tally = None;
15719 runner.state.judge_skipped = false;
15720 runner.state.status = RunStatus::Implementing;
15721 runner.state.config.disk.min_free_bytes = u64::MAX;
15724 let wt = runner.state.candidates[0].worktree.clone();
15725 std::fs::write(
15726 wt.join("interrupted-source"),
15727 "retain implementation work\n",
15728 )
15729 .unwrap();
15730 if committed {
15731 git::rescue_commit(&wt, "implementation checkpoint")
15732 .await
15733 .unwrap();
15734 }
15735 let lock = git::git(
15736 &wt,
15737 &[
15738 "rev-parse",
15739 "--path-format=absolute",
15740 "--git-path",
15741 "index.lock",
15742 ],
15743 )
15744 .await
15745 .unwrap();
15746 if block_rescue {
15747 std::fs::write(&lock, "fixture-owned lock").unwrap();
15748 }
15749 let mut implementer = spec("alpha");
15750 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15751 runner.roles.implementers = vec![implementer];
15752 runner.state.save().unwrap();
15753 let result = runner.implement().await;
15754 assert_eq!(result.is_err(), block_rescue);
15755 assert_eq!(
15756 runner.state.resource_block.as_ref().unwrap().kind,
15757 "disk pressure"
15758 );
15759 assert!(
15760 runner
15761 .state
15762 .events
15763 .iter()
15764 .any(|e| e.message.contains("agent execution interrupted"))
15765 );
15766 assert!(
15767 runner
15768 .state
15769 .dir()
15770 .join("artifacts/candidate-A-recovery/staged.patch")
15771 .exists()
15772 );
15773 if block_rescue {
15774 let saved = RunState::load(&runner.state.id).unwrap();
15775 assert!(saved.candidate_preservation.contains_key(&'A'));
15776 assert!(!saved.candidates[0].empty);
15777 assert_eq!(
15778 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15779 "retain implementation work\n"
15780 );
15781 std::fs::remove_file(lock).unwrap();
15782 runner.state = saved;
15783 } else {
15784 assert_eq!(runner.state.status, RunStatus::Blocked);
15785 assert_eq!(runner.state.candidates[0].commits, 1);
15786 }
15787 runner.roles.implementers.clear();
15790 runner.state.config.disk.min_free_bytes = 0;
15791 runner.state.resource_block = None;
15792 runner.implement().await.unwrap();
15793 assert!(runner.state.candidate_preservation.is_empty());
15794 assert_eq!(runner.state.candidates[0].commits, 1);
15795 assert!(!runner.state.candidates[0].empty);
15796 assert!(git::is_clean(&wt).await.unwrap());
15797 assert!(
15798 crate::run::read_artifact(&runner.state, "cand-A.patch")
15799 .unwrap()
15800 .contains("retain implementation work")
15801 );
15802 }
15803 }
15804
15805 #[tokio::test]
15806 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15807 let (_tmp, mut runner) = preservation_resume_fixture().await;
15808 runner.implement().await.unwrap();
15810 assert!(runner.state.candidate_preservation.is_empty());
15811 assert!(
15812 !runner
15813 .state
15814 .dir()
15815 .join("artifacts/candidate-A-recovery")
15816 .exists()
15817 );
15818 runner.fold_losers().await.unwrap();
15819 assert!(!runner.state.candidates[0].worktree.exists());
15820 for schema in [14, crate::run::SCHEMA] {
15821 runner.state.schema = schema;
15822 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15823 runner.state.save().unwrap();
15824 runner.state = RunState::load(&runner.state.id).unwrap();
15825 runner.implement().await.unwrap();
15826 assert!(runner.state.candidate_preservation.is_empty());
15827 assert!(runner.state.candidates[0].folded);
15828 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15829 }
15830 }
15831
15832 #[tokio::test]
15833 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15834 let (_tmp, mut runner) = preservation_resume_fixture().await;
15835 runner.state.candidates.retain(|c| c.label == 'A');
15836 runner.state.tally = None;
15837 runner.state.judge_skipped = false;
15838 let candidate = &mut runner.state.candidates[0];
15839 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15840 candidate.failed = Some("deployment validation unavailable".into());
15841 std::fs::write(
15842 candidate.worktree.join("repair"),
15843 "future launcher policy\n",
15844 )
15845 .unwrap();
15846 runner
15847 .state
15848 .candidate_preservation
15849 .insert('A', "awaiting persistence".into());
15850 runner.persist_candidates().await.unwrap();
15851 let candidate = &runner.state.candidates[0];
15852 assert!(!candidate.empty);
15853 assert_eq!(candidate.commits, 1);
15854 assert_eq!(candidate.files, 1);
15855 assert!(candidate.failed.is_none());
15856 assert!(candidate.summary.contains("EPERM"));
15857 assert!(
15858 crate::run::read_artifact(&runner.state, "cand-A.patch")
15859 .unwrap()
15860 .contains("future launcher policy")
15861 );
15862 assert!(runner.state.candidate_preservation.is_empty());
15863 }
15864
15865 #[tokio::test]
15866 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15867 let (_tmp, mut runner) = preservation_resume_fixture().await;
15868 let wt = runner.state.candidates[0].worktree.clone();
15869 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15870 runner.state.schema = 14;
15871 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15872 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15873 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15874 for _ in 0..2 {
15875 assert!(runner.implement().await.is_err());
15876 assert_eq!(runner.state.status, RunStatus::Blocked);
15877 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15878 assert!(
15879 runner.state.candidates[0].empty,
15880 "old verdict must remain unusable"
15881 );
15882 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15883 assert!(runner.state.judge_skipped);
15884 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15885 assert!(
15886 crate::run::read_artifact(&runner.state, "cand-A.patch")
15887 .unwrap()
15888 .contains("owned edits")
15889 );
15890 assert!(wt.exists());
15891 assert_eq!(
15892 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15893 .await
15894 .unwrap(),
15895 1
15896 );
15897 runner.state = RunState::load(&runner.state.id).unwrap();
15898 }
15899 }
15900
15901 #[tokio::test]
15902 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15903 let (_tmp, mut runner) = preservation_resume_fixture().await;
15904 let wt = runner.state.candidates[0].worktree.clone();
15905 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15906 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15907 let index_lock = PathBuf::from(
15908 git::git(
15909 &wt,
15910 &[
15911 "rev-parse",
15912 "--path-format=absolute",
15913 "--git-path",
15914 "index.lock",
15915 ],
15916 )
15917 .await
15918 .unwrap(),
15919 );
15920 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15921 runner.state.schema = 14;
15922 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15923 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15924 assert!(runner.implement().await.is_err());
15925 assert!(runner.after_implement().is_err());
15926 assert!(runner.fold_losers().await.is_err());
15927 assert_eq!(
15928 std::fs::read_to_string(wt.join("recovered")).unwrap(),
15929 "failed seat edits"
15930 );
15931 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15932 runner.state = RunState::load(&runner.state.id).unwrap();
15933 std::fs::remove_file(index_lock).unwrap();
15934 assert!(runner.implement().await.is_err());
15936 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15937 assert!(runner.state.candidates[0].empty);
15938 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15939 assert!(
15940 crate::run::read_artifact(&runner.state, "cand-A.patch")
15941 .unwrap()
15942 .contains("failed seat edits")
15943 );
15944 assert!(wt.exists());
15945 }
15946
15947 #[tokio::test]
15948 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15949 let (_tmp, mut runner) = preservation_resume_fixture().await;
15950 let repo = runner.state.repo.clone();
15951 let common = git::git(
15952 &repo,
15953 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15954 )
15955 .await
15956 .unwrap();
15957 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15958 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15959 assert!(runner.fold_losers().await.is_err());
15960 assert!(!runner.state.candidates[0].worktree.exists());
15961 assert!(!runner.state.candidates[0].folded);
15962 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15963 runner.state = RunState::load(&runner.state.id).unwrap();
15964 std::fs::remove_file(branch_lock).unwrap();
15965 let judge = runner.state.worktree_root().join("judge-1");
15966 git::worktree_add_detached(&repo, &judge, "main")
15967 .await
15968 .unwrap();
15969 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15970 runner.roles.judges = vec![spec("judge")];
15971 assert!(runner.fold_losers().await.is_err());
15972 runner.state = RunState::load(&runner.state.id).unwrap();
15973 assert!(
15974 runner.state.candidates[0].folded,
15975 "candidate progress saved before judge failure"
15976 );
15977 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15978 assert!(judge.join("uncommitted").exists());
15979 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15980 runner.fold_losers().await.unwrap();
15981 assert!(!judge.exists());
15982 runner.fold_losers().await.unwrap();
15983 }
15984
15985 #[tokio::test]
15986 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15987 let (_tmp, mut runner) = preservation_resume_fixture().await;
15988 runner.state.candidates.retain(|c| c.label == 'B');
15989 runner.state.tally = None;
15990 runner.state.judge_skipped = false;
15991 runner.state.prep_complete = Some(true);
15992 runner.state.config.graph.advise = false;
15993 runner.state.config.graph.selected_implementer = Some("alpha".into());
15994 let wt = runner.state.candidates[0].worktree.clone();
15995 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15996 runner
15997 .state
15998 .candidate_preservation
15999 .insert('B', "awaiting persistence".into());
16000 let index_lock = git::git(
16001 &wt,
16002 &[
16003 "rev-parse",
16004 "--path-format=absolute",
16005 "--git-path",
16006 "index.lock",
16007 ],
16008 )
16009 .await
16010 .unwrap();
16011 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16012 runner.state.save().unwrap();
16013 assert!(runner.execute_graph().await.is_err());
16014 assert_eq!(runner.state.status, RunStatus::Blocked);
16015 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16016 std::fs::remove_file(index_lock).unwrap();
16017 for _ in 0..2 {
16020 runner.state = RunState::load(&runner.state.id).unwrap();
16021 runner.pause.park();
16022 runner.execute_graph().await.unwrap();
16023 assert_eq!(runner.state.status, RunStatus::Implementing);
16024 assert!(runner.state.candidate_preservation.is_empty());
16025 assert_eq!(runner.state.candidates[0].commits, 2);
16026 assert!(runner.state.tally.is_none());
16027 }
16028 runner.implement().await.unwrap();
16029 assert_eq!(runner.state.status, RunStatus::Judging);
16030 runner.state.status = RunStatus::Blocked;
16032 runner.state.candidates[0].failed = Some("implementation failed".into());
16033 runner.state.save().unwrap();
16034 let error = runner.execute_graph().await.unwrap_err();
16035 assert!(
16036 error
16037 .to_string()
16038 .contains("selected implementation is held")
16039 );
16040 }
16041
16042 #[tokio::test]
16043 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16044 let (_tmp, mut runner) = preservation_resume_fixture().await;
16045 let repo = runner.state.repo.clone();
16046 let candidate = runner.state.candidates[0].clone();
16047 let registered = git::worktree_holding(&repo, &candidate.branch)
16050 .await
16051 .unwrap()
16052 .unwrap();
16053 assert_eq!(
16054 registered.canonicalize().unwrap(),
16055 candidate.worktree.canonicalize().unwrap()
16056 );
16057 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16058 assert_eq!(
16059 git::worktree_holding(&repo, &candidate.branch)
16060 .await
16061 .unwrap(),
16062 Some(registered)
16063 );
16064 assert!(
16065 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16066 "the stale registration initially keeps the branch checked out"
16067 );
16068 runner.fold_losers().await.unwrap();
16069 assert!(runner.state.candidates[0].folded);
16070 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16071 assert_eq!(
16072 git::worktree_holding(&repo, &candidate.branch)
16073 .await
16074 .unwrap(),
16075 None
16076 );
16077 runner.state = RunState::load(&runner.state.id).unwrap();
16078 runner.fold_losers().await.unwrap();
16079 }
16080
16081 #[tokio::test]
16082 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16083 let (_tmp, mut runner) = preservation_resume_fixture().await;
16084 let repo = runner.state.repo.clone();
16085 let candidate = runner.state.candidates[0].clone();
16086 let winner = runner.state.candidates[1].clone();
16087 let registered = git::worktree_holding(&repo, &candidate.branch)
16088 .await
16089 .unwrap()
16090 .unwrap();
16091 assert_eq!(
16092 registered.canonicalize().unwrap(),
16093 candidate.worktree.canonicalize().unwrap()
16094 );
16095 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16096 assert_eq!(
16097 git::worktree_holding(&repo, &candidate.branch)
16098 .await
16099 .unwrap(),
16100 Some(registered)
16101 );
16102 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16103
16104 for _ in 0..2 {
16105 fold_run(&mut runner.state, false, &crate::run::home())
16106 .await
16107 .unwrap();
16108 runner.state = RunState::load(&runner.state.id).unwrap();
16109 assert!(runner.state.candidates[0].folded);
16110 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16111 assert_eq!(
16112 git::worktree_holding(&repo, &candidate.branch)
16113 .await
16114 .unwrap(),
16115 None
16116 );
16117 assert!(!runner.state.candidates[1].folded);
16118 assert!(winner.worktree.exists());
16119 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16120 }
16121 }
16122
16123 #[tokio::test]
16124 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16125 ask_test_home();
16126 let tmp = tempfile::tempdir().unwrap();
16127 let repo = tmp.path().join("repo");
16128 std::fs::create_dir(&repo).unwrap();
16129 init_repo(&repo);
16130 let mut runner = runner_at(RunStatus::Implementing);
16131 runner.state.repo = repo.clone();
16132 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16133 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16134 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16135 let branch = format!("candidate-{label}");
16136 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16137 git::worktree_add_branch(&repo, &wt, &branch, "main")
16138 .await
16139 .unwrap();
16140 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16141 git::git(&wt, &["add", "README.md"]).await.unwrap();
16142 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16143 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16144 runner.state.candidates.push(Candidate {
16145 index,
16146 label,
16147 agent: "alpha".to_owned(),
16148 branch,
16149 worktree: wt,
16150 summary: "completed implementation".to_owned(),
16151 stat: String::new(),
16152 files: 0,
16153 commits: 0,
16154 empty: false,
16155 failed: None,
16156 verified_noop: None,
16157 duration_ms: 42,
16158 folded: false,
16159 });
16160 runner
16161 .state
16162 .candidate_preservation
16163 .insert(label, "awaiting persistence".to_owned());
16164 }
16165 let wt = runner.state.candidates[0].worktree.clone();
16166 let index_lock = git::git(
16167 &wt,
16168 &[
16169 "rev-parse",
16170 "--path-format=absolute",
16171 "--git-path",
16172 "index.lock",
16173 ],
16174 )
16175 .await
16176 .unwrap();
16177 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16178 runner.state.save().unwrap();
16179 assert!(runner.persist_candidates().await.is_err());
16180 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16181 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16182 assert_eq!(
16183 runner.state.candidates[1].commits, 1,
16184 "other results are still collected"
16185 );
16186 assert!(
16187 !runner.state.candidates[0].empty,
16188 "failed rescue is not no change"
16189 );
16190 assert_eq!(
16191 runner.state.candidates[0].summary,
16192 "completed implementation"
16193 );
16194 assert!(runner.after_implement().is_err());
16195 let home = tmp.path().join("fold-home");
16196 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16197 assert!(wt.join("new-file").exists());
16198 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16199 assert!(recovery.join("staged.patch").exists());
16200 assert!(recovery.join("unstaged.patch").exists());
16201 assert_eq!(
16202 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16203 "untracked edits\n"
16204 );
16205 runner.state.begin_pass();
16206 runner.state.end_pass_parked(ParkCause::Upgrade {
16207 from: "1.0".into(),
16208 to: Some("1.1".into()),
16209 });
16210 runner.state.save().unwrap();
16211 runner.state = RunState::load(&runner.state.id).unwrap();
16212 assert!(matches!(
16213 runner.state.passes.last().unwrap().end,
16214 Some(crate::run::PassEnd::Parked {
16215 cause: ParkCause::Upgrade { .. }
16216 })
16217 ));
16218 std::fs::remove_file(index_lock).unwrap();
16219 runner.implement().await.unwrap();
16221 assert!(runner.state.candidate_preservation.is_empty());
16222 assert!(
16223 runner
16224 .state
16225 .candidates
16226 .iter()
16227 .all(|c| c.commits == 1 && !c.empty)
16228 );
16229 assert!(git::is_clean(&wt).await.unwrap());
16230 assert!(
16231 crate::run::read_artifact(&runner.state, "cand-A.patch")
16232 .unwrap()
16233 .contains("later tracked edits")
16234 );
16235 }
16236
16237 #[tokio::test]
16238 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16239 ask_test_home();
16240 let tmp = tempfile::tempdir().unwrap();
16241 let repo = tmp.path().join("repo");
16242 std::fs::create_dir(&repo).unwrap();
16243 init_repo(&repo);
16244 let wt = tmp.path().join("candidate");
16245 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16246 .await
16247 .unwrap();
16248 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16249 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16250 assert!(wt.join("uncommitted").exists());
16251 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16252 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16253 let mut runner = runner_at(RunStatus::Implementing);
16254 runner.state.repo = repo.clone();
16255 runner.state.base_commit = "missing-base".to_owned();
16256 runner.state.candidates.push(Candidate {
16257 index: 0,
16258 label: 'A',
16259 agent: "alpha".into(),
16260 branch: "candidate".into(),
16261 worktree: wt.clone(),
16262 summary: "completed".into(),
16263 stat: String::new(),
16264 files: 0,
16265 commits: 0,
16266 empty: false,
16267 failed: None,
16268 verified_noop: None,
16269 duration_ms: 1,
16270 folded: false,
16271 });
16272 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16273 assert!(
16274 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16275 .await
16276 .is_err()
16277 );
16278 assert!(!runner.state.candidates[0].folded);
16279 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16280 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16281 runner
16282 .state
16283 .candidate_preservation
16284 .insert('A', "pending".into());
16285 assert!(runner.persist_candidates().await.is_err());
16286 assert!(!runner.state.candidates[0].empty);
16287 assert!(!runner.state.candidates[0].folded);
16288 assert!(wt.exists());
16289 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16290 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16291 std::fs::create_dir_all(&patch_path).unwrap();
16292 assert!(
16293 runner.persist_candidates().await.is_err(),
16294 "artifact failure stays pending"
16295 );
16296 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16297 assert!(!runner.state.candidates[0].empty);
16298 std::fs::remove_dir(patch_path).unwrap();
16299 runner.persist_candidates().await.unwrap();
16300 assert!(
16301 runner.state.candidates[0].empty,
16302 "only successful inspection confirms no change"
16303 );
16304 }
16305
16306 fn init_repo(dir: &Path) {
16307 let run = |args: &[&str]| {
16308 let out = std::process::Command::new("git")
16309 .args(args)
16310 .current_dir(dir)
16311 .quiet()
16312 .output()
16313 .expect("spawn git");
16314 assert!(
16315 out.status.success(),
16316 "git {args:?} failed: {}",
16317 String::from_utf8_lossy(&out.stderr)
16318 );
16319 };
16320 run(&["init", "-b", "main"]);
16321 run(&["config", "user.name", "magi test"]);
16322 run(&["config", "user.email", "magi@example.com"]);
16323 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16324 run(&["add", "-A"]);
16325 run(&["commit", "-m", "init"]);
16326 }
16327
16328 fn ask_test_home() {
16336 crate::run::pin_test_home();
16337 }
16338
16339 #[test]
16340 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16341 ask_test_home();
16342 let mut runner = runner_at(RunStatus::Prep);
16343 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16344 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16347 let error = guard
16348 .finish(
16349 &mut runner.state,
16350 Err(
16351 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16352 .into(),
16353 ),
16354 )
16355 .unwrap_err();
16356 assert_eq!(
16357 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16358 std::io::ErrorKind::PermissionDenied
16359 );
16360 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16361 let message = format!("{error:#}");
16362 assert!(message.contains("original refusal"));
16363 assert!(!message.contains("could not save terminal"));
16364 assert_eq!(
16365 error.chain().count(),
16366 2,
16367 "only startup attribution wraps the original error"
16368 );
16369 let saved = RunState::load(&runner.state.id).unwrap();
16370 assert_eq!(saved.status, RunStatus::Blocked);
16371 assert!(!saved.startup.unwrap().complete);
16372 }
16373
16374 #[test]
16375 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16376 ask_test_home();
16377 let mut runner = runner_at(RunStatus::Prep);
16378 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16379 let id = runner.state.id.clone();
16380 drop(guard);
16381 let saved = RunState::load(&id).unwrap();
16382 assert_eq!(saved.status, RunStatus::Blocked);
16383 assert!(!saved.startup.as_ref().unwrap().complete);
16384 assert!(
16385 saved
16386 .startup
16387 .unwrap()
16388 .failure
16389 .unwrap()
16390 .contains("interrupted")
16391 );
16392 assert!(Runner::resume(&id).is_err());
16393 }
16394
16395 #[test]
16396 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16397 ask_test_home();
16398 let mut runner = runner_at(RunStatus::Prep);
16399 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16400 let error = guard
16401 .finish(
16402 &mut runner.state,
16403 Err(
16404 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16405 .into(),
16406 ),
16407 )
16408 .unwrap_err();
16409 assert_eq!(
16410 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16411 std::io::ErrorKind::PermissionDenied
16412 );
16413 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16414 let mut task = crate::queue::Task::new(
16415 "review".into(),
16416 "criteria".into(),
16417 runner.state.repo.clone(),
16418 crate::queue::Source::Human,
16419 );
16420 task.record_startup_failure(failure.clone());
16421 task.record_startup_failure(failure.clone());
16422 assert_eq!(task.startup_failures.len(), 1);
16423 assert!(task.runs.is_empty());
16424 assert_eq!(task.attempts, 0);
16425 }
16426
16427 #[cfg(unix)]
16428 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16429 ask_test_home();
16430 let tmp = tempfile::tempdir().unwrap();
16431 let repo = tmp.path().join("repo");
16432 std::fs::create_dir(&repo).unwrap();
16433 init_repo(&repo);
16434 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16435 let origin = tmp.path().join("origin.git");
16436 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16437 .await
16438 .unwrap();
16439 git::git(
16440 &repo,
16441 &["remote", "add", "origin", origin.to_str().unwrap()],
16442 )
16443 .await
16444 .unwrap();
16445 git::git(&repo, &["checkout", "-b", "fix/retained"])
16446 .await
16447 .unwrap();
16448 std::fs::write(repo.join("retained"), "retained change").unwrap();
16449 git::git(&repo, &["add", "."]).await.unwrap();
16450 git::git(&repo, &["commit", "-m", "retained implementation"])
16451 .await
16452 .unwrap();
16453 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16454 git::git(&repo, &["checkout", "main"]).await.unwrap();
16455 std::fs::write(repo.join("base-change"), "new base").unwrap();
16456 git::git(&repo, &["add", "."]).await.unwrap();
16457 git::git(&repo, &["commit", "-m", "base moved"])
16458 .await
16459 .unwrap();
16460 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16461 git::git(&repo, &["checkout", "fix/retained"])
16462 .await
16463 .unwrap();
16464 let mut runner = runner_at(RunStatus::Voting);
16465 runner.state.repo = repo.clone();
16466 runner.state.base_commit = base;
16467 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16468 runner.state.config.disk.min_free_bytes = 0;
16469 runner.state.config.graph.review_rounds = 1;
16470 runner.state.config.verify.e2e = vec!["true".into()];
16471 runner.state.config.verify.gate = vec!["true".into()];
16472 runner.state.candidates = vec![Candidate {
16473 index: 0,
16474 label: 'A',
16475 agent: "fixture".into(),
16476 branch: "fix/retained".into(),
16477 worktree: repo.clone(),
16478 summary: String::new(),
16479 stat: String::new(),
16480 files: 1,
16481 commits: 1,
16482 empty: false,
16483 failed: None,
16484 verified_noop: None,
16485 duration_ms: 0,
16486 folded: false,
16487 }];
16488 runner.state.tally = Some(Tally {
16489 first_choice: BTreeMap::new(),
16490 borda: BTreeMap::new(),
16491 winner: 'A',
16492 rankings: 0,
16493 unanimous_initial: true,
16494 deliberated: false,
16495 changed_votes: 0,
16496 unanimous_final: true,
16497 tie_break: None,
16498 judges: 0,
16499 present: 0,
16500 quorum: 0,
16501 met_quorum: true,
16502 uncontested: Some("review-only".into()),
16503 });
16504 let mut task = crate::queue::Task::new(
16505 "fixture".into(),
16506 "fixture".into(),
16507 repo,
16508 crate::queue::Source::Human,
16509 );
16510 task.link_run(&runner.state.id);
16511 crate::queue::Queue::open().put(&mut task).unwrap();
16512 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16513 runner.state.base_sync = Some(BaseSync {
16514 tip: String::new(),
16515 behind: 1,
16516 attempts: BASE_SYNC_ROUNDS,
16517 conflict: None,
16518 already_in: None,
16519 });
16520 runner.sync_to_base().await.unwrap();
16521 assert_eq!(runner.state.status, RunStatus::Blocked);
16522 (tmp, runner, head, task.id)
16523 }
16524
16525 #[cfg(unix)]
16526 #[tokio::test]
16527 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16528 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16529 if std::env::var_os(CHILD).is_none() {
16530 let home = tempfile::tempdir().unwrap();
16531 let output = std::process::Command::new(std::env::current_exe().unwrap())
16532 .args([
16533 "--exact",
16534 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16535 "--nocapture",
16536 ])
16537 .env(CHILD, "1")
16538 .env("MAGI_HOME", home.path())
16539 .env("MAGI_WORKER_HOME", home.path())
16540 .env("MAGI_WORKER_ID", "abcdef")
16541 .output()
16542 .unwrap();
16543 assert!(
16544 output.status.success()
16545 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16546 "{}\n{}",
16547 String::from_utf8_lossy(&output.stdout),
16548 String::from_utf8_lossy(&output.stderr)
16549 );
16550 return;
16551 }
16552 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16553 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16554 let home = crate::run::home();
16555 let queue = crate::queue::Queue::open();
16556 let mut owner = queue.get(&task).unwrap();
16557 owner.release_operator("test-release", None);
16558 owner.start(runner.state.id.clone());
16559 queue.put(&mut owner).unwrap();
16560 runner.state.driver_exited = true;
16561 runner.state.task_instruction = Some(owner.instruction.clone());
16562 runner.state.save().unwrap();
16563 let _claim = queue.claim(&task).unwrap();
16564 let reserve = |id: &str| {
16565 let directory = home.join("workers/jobs").join(id);
16566 std::fs::create_dir_all(&directory).unwrap();
16567 let job = crate::worker::Job {
16568 id: id.into(),
16569 sequence: 1,
16570 kind: "task".into(),
16571 key: format!("task:{task}"),
16572 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16573 generation: "fixture-generation".into(),
16574 version: None,
16575 status: crate::worker::Status::Running,
16576 result: None,
16577 pid: Some(std::process::id()),
16578 created_at: 0,
16579 finished_at: None,
16580 };
16581 std::fs::write(
16582 directory.join("job.json"),
16583 serde_json::to_vec(&job).unwrap(),
16584 )
16585 .unwrap();
16586 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16587 };
16588 let _own = reserve("abcdef");
16589 let other = reserve("fedcba");
16590 assert!(
16591 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16592 .await
16593 .is_err()
16594 );
16595 assert!(
16596 RunState::load(&runner.state.id)
16597 .unwrap()
16598 .base_sync_recovery
16599 .authorizations
16600 .is_empty()
16601 );
16602 drop(other);
16603 assert!(crate::daemon::is_working_on(
16604 &home,
16605 &runner.state.id,
16606 Timestamp::now()
16607 ));
16608 assert!(
16609 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16610 );
16611 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16612 .await
16613 .unwrap();
16614 assert!(
16615 RunState::load(&runner.state.id)
16616 .unwrap()
16617 .base_recovery_pending()
16618 );
16619 }
16620
16621 #[cfg(unix)]
16622 #[tokio::test]
16623 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16624 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16625 let home = crate::run::home();
16626 runner.state.driver_pid = Some(std::process::id());
16627 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16628 runner.state.driver_exited = false;
16629 runner.state.begin_pass();
16630 runner.state.save().unwrap();
16631 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16632 driver.record(&runner.state).unwrap();
16633 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16634 drop(driver);
16636 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16637 assert_eq!(
16638 runner.state.liveness_under(&home, false),
16639 crate::run::Liveness::Dead
16640 );
16641 let queue = crate::queue::Queue::open();
16642 let _claim = queue.claim(&task).unwrap();
16643 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16644 .await
16645 .unwrap();
16646 let disk = RunState::load(&runner.state.id).unwrap();
16647 assert!(!disk.driver_exited);
16648 assert!(disk.base_recovery_pending());
16649 }
16650
16651 #[cfg(unix)]
16652 #[tokio::test]
16653 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16654 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16655 if std::env::var_os(CHILD).is_none() {
16656 use std::os::unix::fs::PermissionsExt;
16657 let tmp = tempfile::tempdir().unwrap();
16658 let gh = tmp.path().join("gh");
16659 std::fs::write(&gh, r#"#!/bin/sh
16660case "$1 $2" in
16661 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16662 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16663 *) exit 2 ;;
16664esac
16665"#).unwrap();
16666 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16667 let mut paths = vec![tmp.path().to_path_buf()];
16668 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16669 let output = std::process::Command::new(std::env::current_exe().unwrap())
16670 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16671 .env(CHILD, "1")
16672 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16673 .env("PATH", std::env::join_paths(paths).unwrap())
16674 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16675 .output().unwrap();
16676 assert!(
16677 output.status.success()
16678 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16679 "{}\n{}",
16680 String::from_utf8_lossy(&output.stdout),
16681 String::from_utf8_lossy(&output.stderr)
16682 );
16683 return;
16684 }
16685 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16686 let repo = runner.state.repo.clone();
16687 git::git(&repo, &["checkout", "main"]).await.unwrap();
16688 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16689 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16690 git::git(&repo, &["checkout", "fix/retained"])
16691 .await
16692 .unwrap();
16693 runner.state.pr = Some(crate::run::PrRecord {
16694 url: "https://example.test/project/pull/1".into(),
16695 number: 1,
16696 state: "open".into(),
16697 checks: "unknown".into(),
16698 round: 0,
16699 rounds: 1,
16700 red_at_merge: Vec::new(),
16701 });
16702 runner.sync_to_base().await.unwrap();
16703 assert_eq!(runner.state.status, RunStatus::Blocked);
16704 assert_eq!(
16705 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16706 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16707 );
16708 assert!(runner.state.base_sync_closure_pending());
16709 assert!(crate::handover::can_resume(&runner.state));
16710 let queue = crate::queue::Queue::open();
16711 assert!(!crate::base_sync_recovery::held_task(
16712 &queue,
16713 &queue.get(&task).unwrap()
16714 ));
16715 assert!(runner.state.authorize_base_sync(&task).is_err());
16716 runner.state.base_sync_recovery.stop = None;
16718 runner.state.status = RunStatus::Voting;
16719 runner.state.gate_ran = true;
16720 runner.state.config.graph.review_rounds = 0;
16721 runner.state.save().unwrap();
16722 assert!(crate::handover::can_resume(&runner.state));
16723 runner.state = RunState::load(&runner.state.id).unwrap();
16724 runner.execute_graph().await.unwrap();
16725 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16726 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16727 assert_eq!(
16728 runner.state.base_sync.as_ref().unwrap().attempts,
16729 BASE_SYNC_ROUNDS
16730 );
16731 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16732 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16733 assert!(runner.state.reviews.is_empty());
16734 assert!(runner.state.gate.is_empty());
16735 }
16736
16737 #[cfg(unix)]
16738 #[tokio::test]
16739 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16740 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16741 runner.state.driver_exited = true;
16742 runner.state.save().unwrap();
16743 let home = crate::run::home();
16744 let queue = crate::queue::Queue::open();
16745 let _claim = queue.claim(&task).unwrap();
16746 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16747 assert!(
16748 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16749 .await
16750 .is_err()
16751 );
16752 drop(lease);
16753 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16754 assert!(
16755 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16756 .await
16757 .is_err()
16758 );
16759 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16760 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16761 .await
16762 .unwrap();
16763 let disk = RunState::load(&runner.state.id).unwrap();
16764 assert!(disk.base_recovery_pending());
16765 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16766 assert!(
16767 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16768 .await
16769 .is_err()
16770 );
16771 assert_eq!(
16772 RunState::load(&runner.state.id)
16773 .unwrap()
16774 .base_sync_recovery
16775 .authorizations
16776 .len(),
16777 1
16778 );
16779 let mut owner = queue.get(&task).unwrap();
16780 owner.runs.push("20260101-000000-ffff".into());
16781 queue.put(&mut owner).unwrap();
16782 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16783 .await
16784 .unwrap_err();
16785 assert!(error.to_string().contains("run task ownership changed"));
16786 }
16787
16788 #[cfg(unix)]
16789 #[tokio::test]
16790 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16791 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16792 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16793 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16794 legacy["status"] = serde_json::json!("voting");
16795 runner.state = serde_json::from_value(legacy).unwrap();
16796 runner.state.save().unwrap();
16797 for _ in 0..3 {
16798 runner.execute_graph().await.unwrap();
16799 assert_eq!(runner.state.status, RunStatus::Blocked);
16800 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16801 assert!(!crate::handover::can_resume(&runner.state));
16802 assert_eq!(
16803 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16804 head
16805 );
16806 assert!(runner.state.reviews.is_empty());
16807 assert!(!runner.state.gate_ran);
16808 }
16809 let notice = crate::notices::run_ended(&runner.state).unwrap();
16810 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16811 runner.state.begin_pass();
16812 runner.state.status = RunStatus::Voting;
16813 runner.deliberate().await.unwrap();
16814 runner.vote().await.unwrap();
16815 runner.finish_after_tally().await.unwrap();
16816 assert_eq!(runner.state.status, RunStatus::Blocked);
16817 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16818 assert_eq!(
16819 crate::notices::run_ended(&runner.state).unwrap().since,
16820 notice.since
16821 );
16822 }
16823
16824 #[cfg(unix)]
16825 #[tokio::test]
16826 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16827 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16828 for detail in [
16829 "fix/retained conflicts with origin/main and did not rebase: conflict",
16830 "rebased fix/retained locally but origin refused the push",
16831 "unknown failure",
16832 ] {
16833 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16834 runner.state.record_base_sync_stop(&head);
16835 assert!(runner.state.authorize_base_sync(&task).is_err());
16836 runner.state.status = RunStatus::Voting;
16837 runner.execute_graph().await.unwrap();
16838 assert_eq!(runner.state.status, RunStatus::Blocked);
16839 assert!(!crate::handover::can_resume(&runner.state));
16840 assert_eq!(
16841 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16842 head
16843 );
16844 }
16845 }
16846
16847 #[cfg(unix)]
16848 #[tokio::test]
16849 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16850 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16851 runner.state.authorize_base_sync(&task).unwrap();
16852 assert!(runner.state.consume_base_recovery(&head));
16853 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16854 runner.state.reserve_base_operation(&head, &tip);
16855 runner.state.save().unwrap();
16856 runner.state = RunState::load(&runner.state.id).unwrap();
16857 assert!(!runner.state.consume_base_recovery(&head));
16858 runner.sync_to_base().await.unwrap();
16859 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16860 assert_eq!(runner.state.base_episode_used(), 1);
16861 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16862 assert!(
16863 runner
16864 .state
16865 .base_sync_recovery
16866 .operations
16867 .last()
16868 .unwrap()
16869 .finished
16870 );
16871 assert!(!runner.state.base_sync_blocked());
16872 runner.sync_to_base().await.unwrap();
16873 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16874 }
16875
16876 #[cfg(unix)]
16877 #[tokio::test]
16878 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16879 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16880 let original = runner
16882 .state
16883 .base_sync
16884 .as_ref()
16885 .unwrap()
16886 .conflict
16887 .clone()
16888 .unwrap();
16889 runner.state.base_sync.as_mut().unwrap().conflict = Some(
16890 original
16891 .replace("synchronization(s)", "rebase(s)")
16892 .replace("synchronizing", "rebasing"),
16893 );
16894 runner.state.base_sync_recovery = Default::default();
16895 runner.state.record_base_sync_stop(&head);
16896 assert_eq!(
16897 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16898 crate::base_sync_recovery::StopKind::Exhausted
16899 );
16900 runner.state.authorize_base_sync(&task).unwrap();
16901 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16902 let auth = legacy["base_sync_recovery"]["authorizations"][0]
16903 .as_object_mut()
16904 .unwrap();
16905 auth.remove("interval");
16906 auth.remove("scope");
16907 runner.state = serde_json::from_value(legacy).unwrap();
16908 assert!(runner.state.consume_base_recovery(&head));
16909 assert_eq!(runner.state.base_episode_used(), 3);
16910 runner.sync_to_base().await.unwrap();
16911 assert_eq!(runner.state.base_episode_used(), 4);
16912 runner.state.resource_block = Some(crate::run::ResourceBlock {
16913 node: "gate".into(),
16914 kind: "cache".into(),
16915 detail: "legacy wait".into(),
16916 at: Timestamp::now(),
16917 cause: crate::run::ResourceBlockCause::Unrecorded,
16918 });
16919 runner.state.save().unwrap();
16920 assert!(
16921 runner
16922 .state
16923 .base_sync_recovery
16924 .episodes
16925 .last()
16926 .unwrap()
16927 .boundary
16928 .is_none()
16929 );
16930 runner.state.resource_block = None;
16931 advance_base_fixture(&runner).await;
16932 runner.sync_to_base().await.unwrap();
16933 assert!(runner.state.base_sync_blocked());
16934 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16935 }
16936
16937 #[cfg(unix)]
16938 async fn advance_base_fixture(runner: &Runner) {
16939 let repo = &runner.state.repo;
16940 for args in [
16941 vec!["checkout", "main"],
16942 vec!["commit", "--allow-empty", "-m", "base advanced"],
16943 vec!["push", "origin", "main"],
16944 vec!["checkout", "fix/retained"],
16945 ] {
16946 git::git(repo, &args).await.unwrap();
16947 }
16948 }
16949
16950 #[cfg(unix)]
16951 fn fresh_base_interval_fixture(runner: &mut Runner) {
16952 runner.state.base_sync.as_mut().unwrap().attempts = 0;
16953 runner.state.base_sync.as_mut().unwrap().conflict = None;
16954 runner.state.base_sync_recovery = Default::default();
16955 runner.state.status = RunStatus::Voting;
16956 }
16957
16958 #[cfg(unix)]
16959 #[tokio::test]
16960 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
16961 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
16962 fresh_base_interval_fixture(&mut runner);
16963 runner.state.begin_pass();
16964 runner.sync_to_base().await.unwrap();
16965 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16966 runner.state.resource_block = Some(crate::run::ResourceBlock {
16967 node: "gate".into(),
16968 kind: "build cache".into(),
16969 detail: "busy".into(),
16970 at: Timestamp::now(),
16971 cause: crate::run::ResourceBlockCause::Timeout {
16972 waited_secs: 1,
16973 limit_secs: 1,
16974 },
16975 });
16976 runner.state.save().unwrap();
16977 let queue = crate::queue::Queue::open();
16978 let mut owner = queue.get(&task).unwrap();
16979 runner.state.task_instruction = Some(owner.instruction.clone());
16980 owner.release_operator("fixture-owner", None);
16981 queue.put(&mut owner).unwrap();
16982 let approval = owner.resumption.clone().unwrap();
16983 runner.state.resource_block = None;
16984 runner.state.save().unwrap();
16985 runner.state = RunState::load(&runner.state.id).unwrap();
16986 advance_base_fixture(&runner).await;
16987 runner.sync_to_base().await.unwrap();
16988 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
16989 assert_eq!(runner.state.base_episode_used(), 1);
16990 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16991 assert!(
16992 runner.state.base_sync_recovery.episodes[0]
16993 .boundary
16994 .as_ref()
16995 .unwrap()
16996 .consumed
16997 );
16998 assert!(
16999 !runner
17000 .state
17001 .renew_base_episode(&task, &approval, &head, "base")
17002 );
17003 for _ in 0..3 {
17004 owner.release_operator("fixture-owner", None);
17005 queue.put(&mut owner).unwrap();
17006 runner.state = RunState::load(&runner.state.id).unwrap();
17007 runner.state.begin_pass();
17008 runner.sync_to_base().await.unwrap();
17009 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17010 assert_eq!(runner.state.base_episode_used(), 1);
17011 }
17012 runner.state.resource_block = Some(crate::run::ResourceBlock {
17013 node: "gate".into(),
17014 kind: "disk pressure".into(),
17015 detail: "free space required".into(),
17016 at: Timestamp::now(),
17017 cause: crate::run::ResourceBlockCause::Other,
17018 });
17019 runner.state.end_pass_parked(ParkCause::Interrupt {
17020 reason: "disk pressure".into(),
17021 });
17022 runner.state.save().unwrap();
17023 owner.release_operator("fixture-owner-after-disk", None);
17024 queue.put(&mut owner).unwrap();
17025 runner.state.resource_block = None;
17026 advance_base_fixture(&runner).await;
17027 runner.sync_to_base().await.unwrap();
17028 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17029 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17030 assert_eq!(runner.state.base_episode_used(), 1);
17031 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17032 }
17033
17034 #[cfg(unix)]
17035 #[tokio::test]
17036 async fn base_recovery_continuous_advancement_stops_at_four() {
17037 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17038 fresh_base_interval_fixture(&mut runner);
17039 for spent in 1..=BASE_SYNC_ROUNDS {
17040 runner.sync_to_base().await.unwrap();
17041 assert!(!runner.state.base_sync_blocked());
17042 assert_eq!(runner.state.base_episode_used(), spent);
17043 advance_base_fixture(&runner).await;
17044 }
17045 runner.sync_to_base().await.unwrap();
17046 assert_eq!(
17047 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17048 crate::base_sync_recovery::StopKind::Exhausted
17049 );
17050 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17051 assert!(stop.unverified_sync);
17052 let action = runner.state.base_sync_next_action();
17053 assert!(action.contains("never reached its gate"));
17054 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17055 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17056 runner.state = RunState::load(&runner.state.id).unwrap();
17057 runner.sync_to_base().await.unwrap();
17058 assert_eq!(runner.state.base_episode_used(), 4);
17059 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17060 }
17061
17062 #[cfg(unix)]
17063 #[tokio::test]
17064 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17065 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17066 runner.state.ensure_base_episode();
17067 runner.state.record_base_progress(&head);
17069 runner.state.base_sync_recovery.stop = None;
17070 runner.state.record_base_sync_stop(&head);
17071 assert!(
17072 !runner
17073 .state
17074 .base_sync_recovery
17075 .stop
17076 .as_ref()
17077 .unwrap()
17078 .unverified_sync
17079 );
17080
17081 runner.state.finish_base_operation(Some(&head));
17082 runner.state.gate_ran = true;
17084 runner.state.gate_head = Some("older-head".into());
17085 runner.state.base_sync_recovery.stop = None;
17086 runner.state.record_base_sync_stop(&head);
17087 runner.state.save().unwrap();
17088 runner.state = RunState::load(&runner.state.id).unwrap();
17089 assert!(
17090 runner
17091 .state
17092 .base_sync_recovery
17093 .stop
17094 .as_ref()
17095 .unwrap()
17096 .unverified_sync
17097 );
17098
17099 runner.state.gate_head = Some(head.clone());
17100 runner.state.base_sync_recovery.stop = None;
17101 runner.state.record_base_sync_stop(&head);
17102 assert!(
17103 !runner
17104 .state
17105 .base_sync_recovery
17106 .stop
17107 .as_ref()
17108 .unwrap()
17109 .unverified_sync
17110 );
17111 assert!(
17112 !runner
17113 .state
17114 .base_sync_next_action()
17115 .contains("never reached its gate")
17116 );
17117
17118 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17120 legacy["base_sync_recovery"]["stop"]
17121 .as_object_mut()
17122 .unwrap()
17123 .remove("unverified_sync");
17124 legacy["base_sync_recovery"]["episodes"][0]
17125 .as_object_mut()
17126 .unwrap()
17127 .remove("last_sync_head");
17128 runner.state = serde_json::from_value(legacy).unwrap();
17129 assert!(
17130 !runner
17131 .state
17132 .base_sync_recovery
17133 .stop
17134 .as_ref()
17135 .unwrap()
17136 .unverified_sync
17137 );
17138 assert!(
17139 runner.state.base_sync_recovery.episodes[0]
17140 .last_sync_head
17141 .is_none()
17142 );
17143 }
17144
17145 #[cfg(unix)]
17146 #[tokio::test]
17147 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17148 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17149 runner.state.authorize_base_sync(&task).unwrap();
17150 assert!(
17151 runner
17152 .state
17153 .base_sync_next_action()
17154 .contains("Approval recorded")
17155 );
17156 assert!(
17157 runner
17158 .state
17159 .base_sync_next_action()
17160 .contains("4/4 in episode 0, 4 lifetime")
17161 );
17162 assert!(runner.state.consume_base_recovery(&head));
17163 assert!(
17164 runner
17165 .state
17166 .base_sync_next_action()
17167 .contains("0/4 in episode 1, 4 lifetime")
17168 );
17169 }
17170
17171 #[cfg(unix)]
17172 #[tokio::test]
17173 async fn base_recovery_changed_head_invalidates_without_spending() {
17174 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17175 runner.state.authorize_base_sync(&task).unwrap();
17176 git::git(
17177 &runner.state.repo,
17178 &["commit", "--allow-empty", "-m", "external head change"],
17179 )
17180 .await
17181 .unwrap();
17182 runner.sync_to_base().await.unwrap();
17183 let a = runner
17184 .state
17185 .base_sync_recovery
17186 .authorizations
17187 .last()
17188 .unwrap();
17189 assert!(a.consumed);
17190 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17191 runner.state = RunState::load(&runner.state.id).unwrap();
17192 assert!(
17193 runner
17194 .state
17195 .base_sync_next_action()
17196 .contains("Approval invalidated")
17197 );
17198 assert!(!runner.state.base_recovery_pending());
17199 assert!(runner.state.base_sync_blocked());
17200 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17201 }
17202
17203 #[cfg(unix)]
17204 #[tokio::test]
17205 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17206 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17207 runner.state.driver_exited = true;
17208 runner.state.save().unwrap();
17209 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17210 .await
17211 .unwrap();
17212 runner.state = RunState::load(&runner.state.id).unwrap();
17213 advance_base_fixture(&runner).await;
17214 runner.sync_to_base().await.unwrap();
17215 let a = runner
17216 .state
17217 .base_sync_recovery
17218 .authorizations
17219 .last()
17220 .unwrap();
17221 assert!(
17222 a.consumed
17223 && a.invalidated
17224 .as_deref()
17225 .unwrap()
17226 .contains("fetched base changed")
17227 );
17228 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17229 assert_eq!(
17230 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17231 head
17232 );
17233 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17234 .await
17235 .unwrap();
17236 runner.state = RunState::load(&runner.state.id).unwrap();
17237 runner.sync_to_base().await.unwrap();
17238 assert!(!runner.state.base_sync_blocked());
17239 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17240 }
17241
17242 #[cfg(unix)]
17243 #[tokio::test]
17244 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17245 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17246 runner.state.authorize_base_sync(&task).unwrap();
17247 assert!(runner.state.consume_base_recovery(&head));
17248 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17249 runner.state.reserve_base_operation(&head, &tip);
17250 runner.state.save().unwrap();
17251 let scratch = runner.state.dir().join("base-sync");
17252 assert_eq!(
17253 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17254 .await
17255 .unwrap(),
17256 crate::rebase::Rebased::Applied
17257 );
17258 runner.state = RunState::load(&runner.state.id).unwrap();
17259 runner.sync_to_base().await.unwrap();
17260 assert!(
17261 runner
17262 .state
17263 .base_sync_recovery
17264 .operations
17265 .last()
17266 .unwrap()
17267 .finished
17268 );
17269 assert_eq!(runner.state.base_episode_used(), 1);
17270 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17271 assert!(!runner.state.base_sync_blocked());
17272 }
17273
17274 #[cfg(unix)]
17275 #[tokio::test]
17276 async fn base_recovery_crash_preserves_untracked_files() {
17277 for ignored in [false, true] {
17278 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17279 runner.state.authorize_base_sync(&task).unwrap();
17280 assert!(runner.state.consume_base_recovery(&head));
17281 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17282 runner.state.reserve_base_operation(&head, &tip);
17283 runner.state.save().unwrap();
17284 let scratch = runner.state.dir().join("base-sync");
17285 assert_eq!(
17286 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17287 .await
17288 .unwrap(),
17289 crate::rebase::Rebased::Applied
17290 );
17291 let repo = &runner.state.repo;
17292 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17293 assert_ne!(result, head);
17294 if ignored {
17295 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17296 .await
17297 .unwrap();
17298 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17299 }
17300 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17301 assert!(
17302 git::git_raw(repo, &["diff", "--quiet", &head])
17303 .await
17304 .unwrap()
17305 .ok()
17306 );
17307 runner.state = RunState::load(&runner.state.id).unwrap();
17309 runner.sync_to_base().await.unwrap();
17310 assert!(runner.state.base_sync_blocked());
17311 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17312 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17313 assert_eq!(
17314 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17315 result
17316 );
17317 assert_eq!(
17318 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17319 "retained work\n"
17320 );
17321 assert!(
17322 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17323 .await
17324 .unwrap()
17325 .ok()
17326 );
17327 }
17328 }
17329
17330 #[cfg(unix)]
17331 #[tokio::test]
17332 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17333 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17334 runner.state.authorize_base_sync(&task).unwrap();
17335 assert!(runner.state.consume_base_recovery(&head));
17336 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17337 runner.state.reserve_base_operation(&head, &tip);
17338 runner.state.save().unwrap();
17339 let scratch = runner.state.dir().join("base-sync");
17340 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17341 .await
17342 .unwrap();
17343 git::sync_to_head(&runner.state.repo).await.unwrap();
17344 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17345 git::git(&runner.state.repo, &["add", "foreign"])
17346 .await
17347 .unwrap();
17348 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17349 .await
17350 .unwrap();
17351 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17352 runner.state = RunState::load(&runner.state.id).unwrap();
17353 runner.sync_to_base().await.unwrap();
17354 assert!(runner.state.base_sync_blocked());
17355 assert!(
17356 !runner
17357 .state
17358 .base_sync_recovery
17359 .operations
17360 .last()
17361 .unwrap()
17362 .finished
17363 );
17364 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17365 assert_eq!(
17366 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17367 foreign_head
17368 );
17369 }
17370
17371 #[cfg(unix)]
17372 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17373 {
17374 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17375 fresh_base_interval_fixture(&mut runner);
17376 let repo = &runner.state.repo;
17377 git::git(repo, &["checkout", "main"]).await.unwrap();
17378 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17379 git::git(repo, &["add", "retained"]).await.unwrap();
17380 git::git(repo, &["commit", "-m", "upstream retained change"])
17381 .await
17382 .unwrap();
17383 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17384 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17385 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17386 let scratch = runner.state.dir().join("base-sync");
17387 assert!(matches!(
17388 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17389 .await
17390 .unwrap(),
17391 git::RebaseStart::Conflicted(_)
17392 ));
17393 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17394 agent: "fixture".into(),
17395 paths: vec!["retained".into()],
17396 from: Some(head.clone()),
17397 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17398 index_outside: None,
17399 finished: false,
17400 error: None,
17401 launcher: None,
17402 infra: None,
17403 outage: None,
17404 });
17405 (tmp, runner, head, tip, scratch)
17406 }
17407
17408 #[cfg(unix)]
17409 #[tokio::test]
17410 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17411 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17412 std::fs::write(
17415 scratch.join("retained"),
17416 "upstream and retained resolution\n",
17417 )
17418 .unwrap();
17419 runner.state.resource_block = Some(crate::run::ResourceBlock {
17420 node: "rebase".into(),
17421 kind: "repository coordination".into(),
17422 detail: "busy".into(),
17423 at: Timestamp::now(),
17424 cause: crate::run::ResourceBlockCause::Other,
17425 });
17426 runner.state.save().unwrap();
17427 let newer = git::git(
17430 &runner.state.repo,
17431 &[
17432 "commit-tree",
17433 "main^{tree}",
17434 "-p",
17435 "main",
17436 "-m",
17437 "base advanced during wait",
17438 ],
17439 )
17440 .await
17441 .unwrap();
17442 git::git(
17443 &runner.state.repo,
17444 &[
17445 "push",
17446 "origin",
17447 &format!("{}:refs/heads/main", newer.trim()),
17448 ],
17449 )
17450 .await
17451 .unwrap();
17452 runner.state = RunState::load(&runner.state.id).unwrap();
17453 runner.state.resource_block = None;
17454 runner.sync_to_base().await.unwrap();
17455 assert!(!runner.state.base_sync_blocked());
17456 let op = &runner.state.base_sync_recovery.operations[0];
17457 assert_eq!(op.head, head);
17458 assert_eq!(op.tip, tip);
17459 assert!(op.finished);
17460 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17461 assert_eq!(runner.state.rebase_fixes.len(), 1);
17462 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17463 .await
17464 .unwrap();
17465 assert_ne!(latest, tip);
17466 runner.state = RunState::load(&runner.state.id).unwrap();
17467 runner.sync_to_base().await.unwrap();
17468 assert!(!runner.state.base_sync_blocked());
17469 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17470 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17471 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17472 }
17473
17474 #[cfg(unix)]
17475 #[tokio::test]
17476 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17477 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17478 runner.state.reserve_base_operation(&head, &tip);
17479 runner.state.save().unwrap();
17480 std::fs::write(
17481 scratch.join("retained"),
17482 "upstream and retained resolution\n",
17483 )
17484 .unwrap();
17485 git::git(&scratch, &["add", "retained"]).await.unwrap();
17486 git::git(
17487 &scratch,
17488 &["-c", "core.editor=true", "rebase", "--continue"],
17489 )
17490 .await
17491 .unwrap();
17492 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17493 assert!(
17494 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17495 .await
17496 .unwrap()
17497 .is_empty()
17498 );
17499 assert_eq!(
17500 crate::rebase::finish(
17501 &mut runner.state,
17502 &scratch,
17503 "fix/retained",
17504 &head,
17505 &tip,
17506 &["retained".into()],
17507 ""
17508 )
17509 .await
17510 .unwrap(),
17511 crate::rebase::Rebased::Applied
17512 );
17513 assert!(!scratch.exists());
17514 runner.state = RunState::load(&runner.state.id).unwrap();
17516 let op = &runner.state.base_sync_recovery.operations[0];
17517 assert!(!op.finished);
17518 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17519 runner.sync_to_base().await.unwrap();
17520 assert!(!runner.state.base_sync_blocked());
17521 assert!(runner.state.base_sync_recovery.operations[0].finished);
17522 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17523 assert_eq!(runner.state.rebase_fixes.len(), 1);
17524 let winner = runner.state.winner().unwrap().clone();
17525 let wait = runner.repository_wait();
17526 assert!(
17527 verify_current_head(&mut runner.state, &winner, &wait)
17528 .await
17529 .unwrap()
17530 );
17531 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17532 }
17533
17534 #[cfg(unix)]
17535 #[tokio::test]
17536 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17537 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17538 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17539 runner.state.reserve_base_operation(&head, &tip);
17540 runner.state.save().unwrap();
17541 std::fs::write(
17542 scratch.join("retained"),
17543 "upstream and retained resolution\n",
17544 )
17545 .unwrap();
17546 git::git(&scratch, &["add", "retained"]).await.unwrap();
17547 git::git(
17548 &scratch,
17549 &["-c", "core.editor=true", "rebase", "--continue"],
17550 )
17551 .await
17552 .unwrap();
17553 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17554 assert!(!git::rebase_in_progress(&scratch).await);
17555 assert!(
17556 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17557 .await
17558 .unwrap()
17559 .is_empty()
17560 );
17561 match damage {
17562 Some("record") => {
17563 let path =
17564 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17565 .await
17566 .unwrap();
17567 std::fs::write(scratch.join(path), "").unwrap();
17568 }
17569 Some("foreign") => {
17570 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17571 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17572 git::git(&scratch, &["commit", "-m", "foreign work"])
17573 .await
17574 .unwrap();
17575 }
17576 Some("dirty") => {
17577 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17578 }
17579 _ => {}
17580 }
17581 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17582 runner.state = RunState::load(&runner.state.id).unwrap();
17584 assert!(
17585 runner.state.base_sync_recovery.operations[0]
17586 .validated_head
17587 .is_none()
17588 );
17589 runner.sync_to_base().await.unwrap();
17590 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17591 assert_eq!(runner.state.base_episode_used(), 1);
17592 assert_eq!(runner.state.rebase_fixes.len(), 1);
17593 assert_eq!(
17594 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17595 retained_head
17596 );
17597 if damage.is_some() {
17598 assert!(runner.state.base_sync_blocked());
17599 assert!(scratch.exists());
17600 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17601 assert!(
17602 runner.state.base_sync_recovery.operations[0]
17603 .validated_head
17604 .is_none()
17605 );
17606 } else {
17607 assert!(!runner.state.base_sync_blocked());
17608 assert!(!scratch.exists());
17609 assert!(runner.state.base_sync_recovery.operations[0].finished);
17610 assert_eq!(
17611 runner.state.base_sync_recovery.operations[0]
17612 .validated_head
17613 .as_deref(),
17614 Some(result.as_str())
17615 );
17616 let winner = runner.state.winner().unwrap().clone();
17617 let wait = runner.repository_wait();
17618 assert!(
17619 verify_current_head(&mut runner.state, &winner, &wait)
17620 .await
17621 .unwrap()
17622 );
17623 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17624 runner.state = RunState::load(&runner.state.id).unwrap();
17625 runner.sync_to_base().await.unwrap();
17626 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17627 }
17628 }
17629 }
17630
17631 #[cfg(unix)]
17632 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17633 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17634 runner.state.prep_complete = Some(true);
17636 runner.state.base_sync = None;
17637 runner.state.base_sync_recovery = Default::default();
17638 runner.state.config.graph.gate_fix_rounds = 1;
17639 let mut review = review_round(true, 0, 1, 1, false, true);
17640 review.head = head.clone();
17641 review.verified_head = Some(head.clone());
17642 runner.state.reviews = vec![review];
17643 runner.state.gate = vec![CommandOutcome {
17644 command: "old gate".into(),
17645 code: Some(1),
17646 output_tail: "old failure".into(),
17647 duration_ms: 0,
17648 resource_blocked: false,
17649 }];
17650 runner.state.gate_head = Some(head.clone());
17651 runner.state.gate_ran = true;
17652 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17653 agent: "fixture".into(),
17654 failed: runner.state.gate.clone(),
17655 notes: "spent".into(),
17656 committed: false,
17657 error: None,
17658 }];
17659 runner.state.land_approval = Some(crate::run::LandApproval {
17660 question: "old approval".into(),
17661 head: head.clone(),
17662 });
17663 runner.state.status = RunStatus::Blocked;
17664 (tmp, runner, head)
17665 }
17666
17667 #[cfg(unix)]
17668 #[tokio::test]
17669 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17670 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17671 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17672 let phases = tmp.path().join("phases");
17673 runner.state.config.verify.e2e = vec![format!(
17674 "echo e2e >> '{}'; {}",
17675 phases.display(),
17676 if e2e_ok {
17677 "true"
17678 } else {
17679 "echo new-e2e-failure; false"
17680 }
17681 )];
17682 runner.state.config.verify.gate = vec![format!(
17683 "echo gate >> '{}'; {}",
17684 phases.display(),
17685 if gate_ok {
17686 "true"
17687 } else {
17688 "echo new-gate-failure; false"
17689 }
17690 )];
17691 runner.state.save().unwrap();
17692 runner.state = RunState::load(&runner.state.id).unwrap();
17693 runner.sync_to_base().await.unwrap();
17694 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17695 assert_ne!(head, old);
17696 runner.state = RunState::load(&runner.state.id).unwrap();
17698 runner.execute_graph().await.unwrap();
17699 assert_eq!(runner.state.gate_fixes.len(), 1);
17700 assert_eq!(runner.state.reviews.len(), 1);
17701 assert_eq!(runner.state.reviews[0].head, old);
17702 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17703 assert_eq!(runner.state.verification_history.len(), 1);
17704 let history = &runner.state.verification_history[0];
17705 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17706 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17707 assert!(history.archived_at.is_some());
17708 assert_eq!(history.fixes_spent, Some(1));
17709 assert_eq!(history.gate[0].output_tail, "old failure");
17710 let archive_events: Vec<_> = runner
17711 .state
17712 .events
17713 .iter()
17714 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17715 .collect();
17716 assert_eq!(archive_events.len(), 1);
17717 assert!(archive_events[0].message.contains(&short(&old)));
17718 assert!(archive_events[0].message.contains(&short(&head)));
17719 assert!(
17720 archive_events[0]
17721 .message
17722 .contains("1 gate-fix round(s) already spent stay spent")
17723 );
17724 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17725 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17726 if e2e_ok {
17727 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17728 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17729 } else {
17730 assert!(!runner.state.gate_ran);
17731 }
17732 assert_eq!(
17733 runner.state.status,
17734 if e2e_ok && gate_ok {
17735 RunStatus::Ready
17736 } else {
17737 RunStatus::Blocked
17738 }
17739 );
17740 if !e2e_ok || !gate_ok {
17741 let cause = runner.state.verification_blocking_cause().unwrap();
17742 assert!(cause.contains(&head));
17743 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17744 runner.state.event("land", "rebased successfully");
17746 let notice = crate::notices::run_ended(&runner.state).unwrap();
17747 assert!(!format!("{notice:?}").contains("rebased successfully"));
17748 assert!(runner.state.merge.is_none());
17749 }
17750 let before = std::fs::read_to_string(&phases).unwrap();
17751 runner.state.save().unwrap();
17752 runner.state = RunState::load(&runner.state.id).unwrap();
17753 runner.review_loop().await.unwrap();
17754 runner.gate().await.unwrap();
17755 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17756 assert_eq!(runner.state.verification_history.len(), 1);
17757 let history = &runner.state.verification_history[0];
17758 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17759 assert!(history.archived_at.is_some());
17760 assert_eq!(history.fixes_spent, Some(1));
17761 }
17762 }
17763
17764 #[cfg(unix)]
17765 #[tokio::test]
17766 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17767 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17768 let marker = tmp.path().join("gate-ran");
17769 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17770 runner.review_loop().await.unwrap();
17771 runner.gate().await.unwrap();
17772 assert_eq!(runner.state.status, RunStatus::Blocked);
17773 assert!(!marker.exists());
17774 assert!(runner.state.verification_history.is_empty());
17775 runner.state.gate_head = None;
17776 runner.gate().await.unwrap();
17777 assert!(marker.exists());
17778 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17779 assert_eq!(runner.state.verification_history.len(), 1);
17780 assert_eq!(runner.state.gate_fixes.len(), 1);
17781 }
17782
17783 #[cfg(unix)]
17784 #[tokio::test]
17785 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17786 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17787 runner.sync_to_base().await.unwrap();
17788 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17789 runner.state.reconcile_verification_head(&head);
17790 for outcomes in [
17792 Vec::new(),
17793 vec![CommandOutcome {
17794 command: "true".into(),
17795 code: None,
17796 output_tail: "cache unavailable".into(),
17797 duration_ms: 0,
17798 resource_blocked: true,
17799 }],
17800 ] {
17801 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17802 head: head.clone(),
17803 passed: false,
17804 outcomes,
17805 });
17806 runner.state.gate_ran = false;
17807 runner.state.gate.clear();
17808 runner.state.save().unwrap();
17809 runner.state = RunState::load(&runner.state.id).unwrap();
17810 runner.merge().await.unwrap();
17811 assert!(runner.state.merge.is_none());
17812 runner.review_loop().await.unwrap();
17813 runner.gate().await.unwrap();
17814 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17815 assert!(runner.state.gate_allows_progress());
17816 assert_eq!(runner.state.gate_fixes.len(), 1);
17817 assert!(runner.state.merge.is_none());
17818 }
17819 git::git(
17821 &runner.state.repo,
17822 &["commit", "--allow-empty", "-m", "external update"],
17823 )
17824 .await
17825 .unwrap();
17826 runner.merge().await.unwrap();
17827 assert!(runner.state.merge.is_none());
17828 assert_eq!(runner.state.status, RunStatus::Blocked);
17829 }
17830
17831 #[cfg(unix)]
17832 #[tokio::test]
17833 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17834 for waiting_on_gate in [false, true] {
17835 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17836 runner.sync_to_base().await.unwrap();
17837 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17838 runner.state.reconcile_verification_head(&head);
17839 if waiting_on_gate {
17840 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17841 head: head.clone(),
17842 passed: true,
17843 outcomes: Vec::new(),
17844 });
17845 }
17846 let cache = tmp.path().join("cache");
17847 runner.state.config.verify.e2e =
17848 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17849 runner.state.config.verify.gate = vec!["true".into()];
17850 runner.state.config.graph.timeout_resource_wait = 1;
17851 let owner =
17852 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17853 let guard =
17854 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17855 crate::cache::AcquireOutcome::Acquired(g) => g,
17856 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17857 };
17858 runner.review_loop().await.unwrap();
17859 runner.gate().await.unwrap();
17860 assert!(!runner.state.gate_ran);
17861 assert_eq!(runner.state.gate_fixes.len(), 1);
17862 assert!(runner.state.resource_block.is_some());
17863 runner.merge().await.unwrap();
17864 assert!(runner.state.merge.is_none());
17865 drop(guard);
17866 runner.state = RunState::load(&runner.state.id).unwrap();
17867 runner.state.resource_block = None;
17869 runner.review_loop().await.unwrap();
17870 runner.gate().await.unwrap();
17871 assert!(runner.state.gate_allows_progress());
17872 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17873 assert_eq!(runner.state.gate_fixes.len(), 1);
17874 }
17875 }
17876
17877 #[cfg(unix)]
17878 #[tokio::test]
17879 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17880 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17881 runner.sync_to_base().await.unwrap();
17882 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17883 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17884 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17886 runner.review_loop().await.unwrap();
17887 runner.gate().await.unwrap();
17888 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17889 assert_ne!(before, after);
17890 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17891 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17892 assert!(!runner.state.gate_allows_progress());
17893 runner.merge().await.unwrap();
17894 assert!(runner.state.merge.is_none());
17895 assert!(
17896 runner
17897 .state
17898 .verification_history
17899 .iter()
17900 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17901 );
17902
17903 runner.state.reviews[0].clean = false;
17905 runner.state.reviews[0].answered = 0;
17906 runner.state.config.verify.e2e = vec!["true".into()];
17907 runner.review_loop().await.unwrap();
17908 runner.gate().await.unwrap();
17909 runner.merge().await.unwrap();
17910 assert_eq!(runner.state.status, RunStatus::Blocked);
17911 assert!(runner.state.merge.is_none());
17912 assert!(
17913 runner
17914 .state
17915 .verification_blocking_cause()
17916 .unwrap()
17917 .contains("review blocked")
17918 );
17919 }
17920
17921 #[cfg(unix)]
17922 #[tokio::test]
17923 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
17924 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
17925 runner.state.gate_head = None;
17926 runner.state.config.verify.gate =
17927 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
17928 runner.review_loop().await.unwrap();
17929 runner.gate().await.unwrap();
17930 assert_ne!(
17931 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17932 old
17933 );
17934 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
17935 assert!(!runner.state.gate_allows_progress());
17936 assert_eq!(runner.state.status, RunStatus::Blocked);
17937 runner.merge().await.unwrap();
17938 assert!(runner.state.merge.is_none());
17939 }
17940
17941 #[cfg(unix)]
17942 #[tokio::test]
17943 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
17944 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17945 let mut old_review = review_round(true, 0, 1, 1, false, true);
17946 old_review.head = head.clone();
17947 old_review.verified_head = Some(head.clone());
17948 runner.state.reviews = vec![old_review];
17949 runner.state.gate = vec![CommandOutcome {
17950 command: "old gate".into(),
17951 code: None,
17952 output_tail: "timed out after 2400s".into(),
17953 duration_ms: 2_400_000,
17954 resource_blocked: false,
17955 }];
17956 runner.state.gate_ran = true;
17957 runner.state.gate_head = Some(head.clone());
17958 runner.state.land_approval = Some(crate::run::LandApproval {
17959 question: "old-head-approval".into(),
17960 head: head.clone(),
17961 });
17962 runner.state.authorize_base_sync(&task).unwrap();
17963 runner.sync_to_base().await.unwrap();
17964 assert!(!runner.state.base_sync_blocked());
17965 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17966 assert_ne!(head, fresh);
17967 assert_eq!(runner.state.reviews.len(), 1);
17968 assert!(!runner.state.gate_ran);
17969 let saved = &runner.state.base_sync_recovery.authorizations[0];
17970 assert_eq!(saved.reviews[0].head, head);
17971 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
17972 assert!(saved.gate_ran);
17973 assert!(!saved.gate[0].ok());
17974 runner.review_loop().await.unwrap();
17975 runner.gate().await.unwrap();
17976 assert_eq!(runner.state.reviews[0].head, head);
17977 assert_eq!(
17978 runner.state.reviews[0].verified_head.as_deref(),
17979 Some(head.as_str())
17980 );
17981 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
17982 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17983 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
17984 assert!(runner.state.gate_ran);
17985 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17986 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
17987 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
17988 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17989 assert_eq!(
17990 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
17991 "retained change"
17992 );
17993 }
17994
17995 #[cfg(unix)]
17996 #[tokio::test]
17997 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
17998 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17999 let queue = crate::queue::Queue::open();
18000 let mut owner = queue.get(&task).unwrap();
18001 owner.release_operator("test-resume", None);
18002 queue.put(&mut owner).unwrap();
18003 let generation = owner.resumption.as_ref().unwrap().generation;
18004 runner.state.task_instruction = Some(owner.instruction.clone());
18005 runner.state.driver_exited = true;
18006 runner.state.save().unwrap();
18007 let home = crate::run::home();
18008 let _claim = queue.claim(&task).unwrap();
18009 owner.resumption.as_mut().unwrap().at =
18010 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18011 queue.put(&mut owner).unwrap();
18012 assert!(
18013 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18014 .await
18015 .is_err(),
18016 "an approval from before the stop grants nothing"
18017 );
18018 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18019 queue.put(&mut owner).unwrap();
18020 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18021 .await
18022 .unwrap();
18023 assert!(
18024 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18025 .await
18026 .is_err()
18027 );
18028 runner.state = RunState::load(&runner.state.id).unwrap();
18029 assert_eq!(
18030 runner.state.base_sync_recovery.authorizations[0].generation,
18031 Some(generation)
18032 );
18033 runner.sync_to_base().await.unwrap();
18034 let repo = runner.state.repo.clone();
18035 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18036 git::git(&repo, &["checkout", "main"]).await.unwrap();
18037 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18038 git::git(&repo, &["add", "."]).await.unwrap();
18039 git::git(&repo, &["commit", "-m", "move base again"])
18040 .await
18041 .unwrap();
18042 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18043 git::git(&repo, &["checkout", "fix/retained"])
18044 .await
18045 .unwrap();
18046 runner.sync_to_base().await.unwrap();
18047 assert!(!runner.state.base_sync_blocked());
18048 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18049 assert_eq!(runner.state.base_episode_used(), 2);
18050 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18051 assert_ne!(fresh, head);
18052 assert!(!runner.state.base_recovery_pending());
18053 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18055 queue.put(&mut owner).unwrap();
18056 assert!(
18057 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18058 .await
18059 .is_err()
18060 );
18061 runner.state = RunState::load(&runner.state.id).unwrap();
18062 runner.sync_to_base().await.unwrap();
18063 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18064 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18065 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18066 }
18067
18068 #[cfg(unix)]
18069 #[tokio::test]
18070 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18071 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18072 runner.state.base_sync = None;
18073 runner.state.status = RunStatus::Blocked;
18074 let mut second = runner.state.candidates[0].clone();
18075 second.label = 'B';
18076 runner.state.candidates.push(second);
18077 assert!(runner.state.votes.is_empty());
18078 assert!(runner.state.deliberation.is_empty());
18079 runner.deliberate().await.unwrap();
18080 runner.vote().await.unwrap();
18081 assert_eq!(runner.state.status, RunStatus::Blocked);
18082 assert!(runner.state.votes.is_empty());
18083 assert!(runner.state.deliberation.is_empty());
18084 }
18085
18086 fn runner_at(status: RunStatus) -> Runner {
18089 let mut state = RunState::new(
18090 PathBuf::from("/nonexistent/repo"),
18091 "main".to_owned(),
18092 "deadbeef".to_owned(),
18093 "task".to_owned(),
18094 Config::default(),
18095 );
18096 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18099 state.id = format!(
18100 "20000101-000000-{:04x}",
18101 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18102 );
18103 state.status = status;
18104 Runner {
18105 state,
18106 roles: ResolvedRoles {
18107 implementers: Vec::new(),
18108 judges: Vec::new(),
18109 reviewers: Vec::new(),
18110 fixer: None,
18111 conductor: conductor(),
18112 implementer_roster: Vec::new(),
18113 judge_roster: Vec::new(),
18114 reviewer_roster: Vec::new(),
18115 },
18116 sem: Arc::new(Semaphore::new(1)),
18117 pause: Pause::new(),
18118 interrupt: Pause::new(),
18119 }
18120 }
18121
18122 #[tokio::test]
18123 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18124 crate::run::pin_test_home();
18125 let mut runner = runner_at(RunStatus::Gating);
18126 runner.state.save().unwrap();
18127 let id = runner.state.id.clone();
18128 let passes = runner.state.passes.len();
18129 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18130 let mut claiming = Box::pin(runner.claim_driver());
18131 std::future::poll_fn(|cx| {
18132 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18133 std::task::Poll::Ready(())
18134 })
18135 .await;
18136 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18137 drop(owner);
18138 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18139 .await
18140 .unwrap()
18141 .unwrap();
18142 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18143 assert!(execution.runner.state.resource_block.is_none());
18144 assert_eq!(
18145 execution.runner.state.liveness(false),
18146 crate::run::Liveness::Live
18147 );
18148 drop(execution);
18149 assert!(RunState::load(&id).unwrap().driver_exited);
18150 }
18151
18152 #[tokio::test]
18153 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18154 crate::run::pin_test_home();
18155 let mut runner = runner_at(RunStatus::Gating);
18156 runner.state.save().unwrap();
18157 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18160 assert!(runner.claim_driver().await.is_err());
18161 assert!(runner.state.driver_exited);
18162 assert_eq!(runner.state.driver_pid, None);
18163 let disk = RunState::load(&runner.state.id).unwrap();
18164 assert!(disk.driver_exited);
18165 assert_eq!(disk.driver_pid, None);
18166 assert_eq!(disk.status, RunStatus::Gating);
18167 }
18168
18169 #[tokio::test]
18170 async fn an_early_graph_error_releases_its_driver() {
18171 crate::run::pin_test_home();
18172 let mut runner = runner_at(RunStatus::Blocked);
18173 runner.state.config.graph.selected_implementer = Some("selected".into());
18174 runner.state.save().unwrap();
18175 assert!(runner.execute().await.is_err());
18176 let disk = RunState::load(&runner.state.id).unwrap();
18177 assert!(disk.driver_exited);
18178 assert_eq!(disk.driver_pid, None);
18179 assert_eq!(disk.status, RunStatus::Blocked);
18180 }
18181
18182 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18183 crate::run::MergeRepair {
18184 question: "repair-question".into(),
18185 head: "old-head".into(),
18186 pr: "https://github.com/o/r/pull/42".into(),
18187 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18188 gate: vec![outcome(Some(0), false)],
18189 gate_head: Some("old-head".into()),
18190 gate_fixes: Vec::new(),
18191 pre_gate: Vec::new(),
18192 phase,
18193 failure: None,
18194 }
18195 }
18196
18197 #[tokio::test]
18198 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18199 crate::run::pin_test_home();
18200 let mut runner = runner_at(RunStatus::Landing);
18201 runner.state.merge_repairs.push(merge_repair_request(
18202 crate::run::MergeRepairPhase::Requested,
18203 ));
18204 runner.state.save().unwrap();
18205 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18206 assert!(
18207 runner
18208 .execute()
18209 .await
18210 .unwrap_err()
18211 .to_string()
18212 .contains("already running")
18213 );
18214 assert_eq!(
18215 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18216 crate::run::MergeRepairPhase::Requested
18217 );
18218 drop(claim);
18219 let mut disk = runner.state.clone();
18220 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18221 disk.save().unwrap();
18222 assert!(
18223 runner
18224 .execute()
18225 .await
18226 .unwrap_err()
18227 .to_string()
18228 .contains("decision changed")
18229 );
18230 assert_eq!(
18231 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18232 crate::run::MergeRepairPhase::Complete
18233 );
18234 }
18235
18236 #[test]
18237 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18238 crate::run::pin_test_home();
18239 let mut runner = runner_at(RunStatus::Landing);
18240 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18241 runner.state.reviews = request.reviews.clone();
18242 runner.state.gate = request.gate.clone();
18243 runner.state.gate_ran = true;
18244 runner.state.land_approval = Some(crate::run::LandApproval {
18245 question: "old".into(),
18246 head: "old-head".into(),
18247 });
18248 runner.state.merge_repairs.push(request);
18249 runner.begin_merge_repair_review(0).unwrap();
18250 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18251 assert!(!runner.state.gate_status().ok());
18252 assert!(runner.state.land_approval.is_none());
18253 let disk = RunState::load(&runner.state.id).unwrap();
18254 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18255 assert!(disk.merge_repairs[0].gate[0].ok());
18256 assert_eq!(
18257 disk.merge_repairs[0].phase,
18258 crate::run::MergeRepairPhase::Reviewing
18259 );
18260 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18261 assert!(disk.reviews.is_empty());
18262 }
18263
18264 #[tokio::test]
18265 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18266 crate::run::pin_test_home();
18267 for phase in [
18268 crate::run::MergeRepairPhase::Fixing,
18269 crate::run::MergeRepairPhase::Requested,
18270 ] {
18271 let mut runner = runner_at(RunStatus::Landing);
18272 runner.state.config.graph.land_rounds = 0;
18273 runner.state.merge_repairs.push(merge_repair_request(phase));
18274 runner.state.save().unwrap();
18275 assert!(!runner.repair_merge_review().await.unwrap());
18276 assert_eq!(runner.state.status, RunStatus::Blocked);
18277 assert_eq!(
18278 runner.state.merge_repairs[0].phase,
18279 crate::run::MergeRepairPhase::Failed
18280 );
18281 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18282 assert!(
18283 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18284 "interrupted"
18285 } else {
18286 "budget exhausted"
18287 })
18288 );
18289 assert!(runner.state.merge.is_none());
18290 }
18291 }
18292
18293 #[test]
18297 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18298 crate::run::pin_test_home();
18299 let mut runner = runner_at(RunStatus::Implementing);
18300 let interrupt = Pause::new();
18301 runner.watch_interrupt(interrupt.clone());
18302
18303 interrupt.park_because("task a1b2 asked to run first");
18304
18305 assert!(runner.park_here().expect("park_here"));
18306 assert!(runner.state.parked);
18307 let last = runner.state.events.last().expect("a park event");
18308 assert_eq!(last.node, "park");
18309 assert!(
18310 last.message.contains("task a1b2 asked to run first"),
18311 "expected the interrupt reason in {:?}",
18312 last.message
18313 );
18314 }
18315
18316 #[test]
18319 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18320 crate::run::pin_test_home();
18321 let mut runner = runner_at(RunStatus::Implementing);
18322 runner.state.begin_pass();
18323 let shutdown = Pause::new();
18324 runner.on_pause(shutdown.clone());
18325 let upgrade = ParkCause::Upgrade {
18326 from: "0.121.0".to_owned(),
18327 to: Some("0.121.1".to_owned()),
18328 };
18329 shutdown.park_for(upgrade.clone());
18330 shutdown.park_for(ParkCause::Operator);
18331
18332 assert!(runner.park_here().expect("park_here"));
18333 assert_eq!(
18334 runner.state.passes[0].end,
18335 Some(crate::run::PassEnd::Parked { cause: upgrade })
18336 );
18337 let last = runner.state.events.last().expect("a park event");
18338 assert!(last.message.contains("0.121.0"), "{}", last.message);
18339 }
18340
18341 #[tokio::test]
18342 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18343 crate::run::pin_test_home();
18344 let mut runner = runner_at(RunStatus::Reviewing);
18345 runner.state.save().unwrap();
18346 let mut execution = runner.claim_driver().await.expect("first execution");
18347 assert_eq!(execution.runner.state.passes.len(), 1);
18348 execution
18349 .driver
18350 .finish(&mut execution.runner.state)
18351 .await
18352 .unwrap();
18353 }
18354
18355 #[tokio::test]
18356 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18357 crate::run::pin_test_home();
18358 let mut runner = runner_at(RunStatus::Reviewing);
18359 runner.state.config.agents = vec![conductor()];
18360 runner.state.config.graph.timeout_resource_wait = 0;
18361 runner.state.save().unwrap();
18362 let id = runner.state.id.clone();
18363 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18364 assert!(runner.claim_driver().await.is_err());
18365 drop(owner);
18366 drop(runner);
18367 let mut resumed = Runner::resume(&id).unwrap();
18368 assert_eq!(resumed.state.driver_claimed, Some(false));
18369 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18370 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18371 let mut execution = resumed
18372 .claim_driver()
18373 .await
18374 .expect("retry initial claim after reload");
18375 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18376 assert_eq!(execution.runner.state.passes.len(), 1);
18377 execution
18378 .driver
18379 .finish(&mut execution.runner.state)
18380 .await
18381 .unwrap();
18382 }
18383
18384 #[tokio::test]
18385 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18386 crate::run::pin_test_home();
18387 let mut runner = runner_at(RunStatus::Reviewing);
18388 runner.state.config.agents = vec![conductor()];
18389 runner.state.save().unwrap();
18390 let id = runner.state.id.clone();
18391 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18392 let mut resumed = Runner::resume(&id).unwrap();
18393 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18394 assert!(resumed.claim_driver().await.is_err());
18395 assert!(RunState::load(&id).unwrap().passes.is_empty());
18396 drop(lease);
18397 let mut execution = resumed.claim_driver().await.unwrap();
18398 execution
18399 .driver
18400 .finish(&mut execution.runner.state)
18401 .await
18402 .unwrap();
18403 }
18404
18405 #[tokio::test]
18406 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18407 crate::run::pin_test_home();
18408 let mut runner = runner_at(RunStatus::Reviewing);
18409 let state = &mut runner.state;
18410 state.config.agents = vec![conductor()];
18411 state.driver_claimed = None;
18412 state.driver_pid = Some(std::process::id());
18413 state.driver_started_at = None;
18414 state.begin_pass();
18415 state.save().unwrap();
18416 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18417 std::fs::write(
18418 state.dir().join("driver.json"),
18419 serde_json::to_vec(&serde_json::json!({
18420 "pid": state.driver_pid, "started_at": null,
18421 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18422 }))
18423 .unwrap(),
18424 )
18425 .unwrap();
18426 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18427 drop(lease);
18428 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18429 let mut resumed = Runner::resume(&state.id).unwrap();
18430 assert!(!resumed.state.driver_exited);
18431 let mut execution = resumed
18432 .claim_driver()
18433 .await
18434 .expect("lease proves previous execution ended");
18435 assert_eq!(execution.runner.state.passes.len(), 2);
18436 execution
18437 .driver
18438 .finish(&mut execution.runner.state)
18439 .await
18440 .unwrap();
18441 }
18442
18443 #[tokio::test]
18444 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18445 crate::run::pin_test_home();
18446 let mut runner = runner_at(RunStatus::Reviewing);
18447 let mut json = serde_json::to_value(&runner.state).unwrap();
18448 json.as_object_mut().unwrap().remove("driver_claimed");
18449 runner.state = serde_json::from_value(json).unwrap();
18450 assert_eq!(runner.state.driver_claimed, None);
18451 runner.state.save().unwrap();
18452 let mut execution = runner
18453 .claim_driver()
18454 .await
18455 .expect("the execution lease, not a missing marker, excludes a second driver");
18456 assert_eq!(execution.runner.state.passes.len(), 1);
18457 execution
18458 .driver
18459 .finish(&mut execution.runner.state)
18460 .await
18461 .unwrap();
18462 }
18463
18464 #[tokio::test]
18465 async fn driver_admission_refuses_a_proven_live_driver() {
18466 crate::run::pin_test_home();
18467 let mut runner = runner_at(RunStatus::Reviewing);
18468 let pid = std::process::id();
18469 runner.state.driver_pid = Some(pid);
18470 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18471 assert!(runner.state.driver_started_at.is_some());
18472 runner.state.save().unwrap();
18473 let error = runner
18474 .claim_driver()
18475 .await
18476 .err()
18477 .expect("a live driver must refuse");
18478 assert!(error.to_string().contains("live driver"), "{error:#}");
18479 let saved = RunState::load(&runner.state.id).unwrap();
18480 assert_eq!(saved.driver_pid, Some(pid));
18481 assert_eq!(saved.status, RunStatus::Reviewing);
18482 assert!(saved.passes.is_empty());
18483 }
18484
18485 #[test]
18486 fn parking_resource_halt_persists_the_pass_once() {
18487 crate::run::pin_test_home();
18488 let mut runner = runner_at(RunStatus::Reviewing);
18489 runner.state.begin_pass();
18490 runner.state.resource_block = Some(crate::run::ResourceBlock {
18491 node: "fix".into(),
18492 kind: "shared build cache".into(),
18493 detail: "cancelled".into(),
18494 at: Timestamp::now(),
18495 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18496 });
18497 let cause = ParkCause::Upgrade {
18498 from: "old".into(),
18499 to: Some("new".into()),
18500 };
18501 runner.pause.park_for(cause.clone());
18502 runner.stop_for_resource().unwrap();
18505 assert!(runner.resource_halt().unwrap());
18506 assert!(runner.park_here().unwrap());
18507 let saved = RunState::load(&runner.state.id).unwrap();
18508 assert!(saved.parked);
18509 assert_eq!(saved.status, RunStatus::Reviewing);
18510 assert!(saved.passes[0].ended_at.is_some());
18511 assert_eq!(
18512 saved.passes[0].end,
18513 Some(crate::run::PassEnd::Parked {
18514 cause: cause.clone()
18515 })
18516 );
18517 assert_eq!(
18518 saved.resource_block.unwrap().cause,
18519 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18520 );
18521 assert_eq!(
18522 saved
18523 .events
18524 .iter()
18525 .filter(|event| event.node == "park")
18526 .count(),
18527 1
18528 );
18529 }
18530
18531 #[test]
18539 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18540 crate::run::pin_test_home();
18541 let mut runner = runner_at(RunStatus::Implementing);
18542 let shutdown = Pause::new();
18543 runner.on_pause(shutdown.clone());
18544 let interrupt = Pause::new();
18545 runner.watch_interrupt(interrupt.clone());
18546
18547 assert!(!runner.park_here().expect("park_here"));
18549 assert!(!runner.state.parked);
18550
18551 interrupt.park_because("test");
18553 assert!(!shutdown.parked());
18554 assert!(runner.park_here().expect("park_here"));
18555 }
18556
18557 #[tokio::test]
18571 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18572 crate::run::pin_test_home();
18573 let mut runner = runner_at(RunStatus::Implementing);
18574 let interrupt = Pause::new();
18575 runner.watch_interrupt(interrupt.clone());
18576
18577 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18578 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18579
18580 let node = async move {
18584 started_tx.send(()).expect("send started");
18585 finish_rx.await.expect("recv finish");
18586 "node finished"
18587 };
18588
18589 let interrupter = async move {
18590 started_rx.await.expect("recv started");
18591 interrupt.park_because("higher-priority task waiting");
18593 tokio::task::yield_now().await;
18597 finish_tx.send(()).expect("send finish");
18598 };
18599
18600 let (node_result, ()) = tokio::join!(node, interrupter);
18601 assert_eq!(
18602 node_result, "node finished",
18603 "the in-flight call ran to completion"
18604 );
18605
18606 assert!(runner.park_here().expect("park_here"));
18609 assert!(runner.state.parked);
18610 }
18611
18612 #[test]
18618 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18619 crate::run::pin_test_home();
18620 let mut runner = runner_at(RunStatus::Judging);
18621 runner.state.config.agents = vec![conductor()];
18625 runner.state.candidates = vec![Candidate {
18626 index: 0,
18627 label: 'A',
18628 agent: "alpha".to_owned(),
18629 branch: "magi/x/A".to_owned(),
18630 worktree: PathBuf::from("/nonexistent/worktree"),
18631 summary: "did the thing".to_owned(),
18632 stat: "1 file changed".to_owned(),
18633 files: 1,
18634 commits: 1,
18635 empty: false,
18636 failed: None,
18637 verified_noop: None,
18638 duration_ms: 1234,
18639 folded: false,
18640 }];
18641 let run_id = runner.state.id.clone();
18642
18643 let interrupt = Pause::new();
18644 runner.watch_interrupt(interrupt.clone());
18645 interrupt.park_because("task c3d4 asked to run first");
18646 assert!(runner.park_here().expect("park_here"));
18647
18648 let resumed = Runner::resume(&run_id).expect("resume");
18649 assert_eq!(resumed.state.candidates.len(), 1);
18650 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18651 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18652 assert_eq!(resumed.state.status, runner.state.status);
18653 assert!(
18654 resumed.state.parked,
18655 "still parked until `execute` actually walks the graph again"
18656 );
18657 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18658 }
18659
18660 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18662 let mut q = ask::Question::new(
18663 run.to_owned(),
18664 "implement".to_owned(),
18665 "impl-A".to_owned(),
18666 "Which storage backend should the cache use?".to_owned(),
18667 String::new(),
18668 vec!["SQLite".to_owned(), "Redis".to_owned()],
18669 );
18670 store.put(&mut q).unwrap();
18671 q
18672 }
18673
18674 #[test]
18675 fn a_failed_runs_open_question_is_abandoned() {
18676 ask_test_home();
18677 let store = ask::Questions::open();
18678 let mut runner = runner_at(RunStatus::Failed);
18679 let run = runner.state.id.clone();
18680 let q = ask_open_question(&store, &run);
18681
18682 runner.settle_questions();
18683
18684 let back = store.get(&q.id).unwrap();
18685 assert!(
18686 !back.status.open(),
18687 "the seat that asked died with the run; nobody is left to read an answer"
18688 );
18689 assert!(
18690 back.detail.contains(&run) && back.detail.contains("failed"),
18691 "the reason names what the run became, not just that it is gone: {}",
18692 back.detail
18693 );
18694 }
18695
18696 #[test]
18697 fn a_merged_runs_open_question_is_abandoned_too() {
18698 ask_test_home();
18699 let store = ask::Questions::open();
18700 for status in [RunStatus::Merged, RunStatus::Ready] {
18703 let mut runner = runner_at(status);
18704 let run = runner.state.id.clone();
18705 let q = ask_open_question(&store, &run);
18706
18707 runner.settle_questions();
18708
18709 let back = store.get(&q.id).unwrap();
18710 assert!(
18711 !back.status.open(),
18712 "{status:?} run's question must not outlive the run"
18713 );
18714 }
18715 }
18716
18717 #[test]
18718 fn a_still_resumable_runs_open_question_is_left_alone() {
18719 ask_test_home();
18720 let store = ask::Questions::open();
18721 for status in [RunStatus::Blocked, RunStatus::Stalled] {
18727 let mut runner = runner_at(status);
18728 let run = runner.state.id.clone();
18729 let q = ask_open_question(&store, &run);
18730
18731 runner.settle_questions();
18732
18733 let back = store.get(&q.id).unwrap();
18734 assert!(
18735 back.status.open(),
18736 "{status:?} is still alive; the question must still be waiting"
18737 );
18738 }
18739 }
18740
18741 #[test]
18742 fn settle_questions_never_touches_an_already_answered_question() {
18743 ask_test_home();
18744 let store = ask::Questions::open();
18745 let mut runner = runner_at(RunStatus::Failed);
18746 let run = runner.state.id.clone();
18747 let mut q = ask_open_question(&store, &run);
18748 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18749 .unwrap();
18750 store.put(&mut q).unwrap();
18751
18752 runner.settle_questions();
18757 runner.settle_questions();
18758
18759 let back = store.get(&q.id).unwrap();
18760 assert_eq!(
18761 back.status,
18762 ask::QuestionStatus::Answered,
18763 "a real answer is a decision on record, never overwritten by a sweep"
18764 );
18765 }
18766
18767 #[tokio::test]
18778 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
18779 crate::run::pin_test_home();
18780 let tmp = tempfile::tempdir().expect("tempdir");
18781 let repo = tmp.path().join("repo");
18782 std::fs::create_dir_all(&repo).unwrap();
18783 init_repo(&repo);
18784
18785 let mut config = Config::default();
18786 config.graph.worktree_root = Some(tmp.path().join("wt"));
18787
18788 let mut state = RunState::new(
18789 repo.clone(),
18790 "main".to_owned(),
18791 "deadbeef".to_owned(),
18792 "task".to_owned(),
18793 config,
18794 );
18795 let root = state.worktree_root();
18796 let wt_a = root.join("cand-A");
18797 let wt_b = root.join("cand-B");
18798 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
18799 .await
18800 .expect("worktree A");
18801 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
18802 .await
18803 .expect("worktree B");
18804
18805 state.candidates = vec![
18806 Candidate {
18807 index: 0,
18808 label: 'A',
18809 agent: "alpha".to_owned(),
18810 branch: "magi/x/A".to_owned(),
18811 worktree: wt_a.clone(),
18812 summary: String::new(),
18813 stat: String::new(),
18814 files: 0,
18815 commits: 0,
18816 empty: false,
18817 failed: None,
18818 verified_noop: None,
18819 duration_ms: 0,
18820 folded: false,
18821 },
18822 Candidate {
18823 index: 1,
18824 label: 'B',
18825 agent: "beta".to_owned(),
18826 branch: "magi/x/B".to_owned(),
18827 worktree: wt_b.clone(),
18828 summary: String::new(),
18829 stat: String::new(),
18830 files: 0,
18831 commits: 0,
18832 empty: false,
18833 failed: None,
18834 verified_noop: None,
18835 duration_ms: 0,
18836 folded: false,
18837 },
18838 ];
18839 state.tally = Some(Tally {
18840 first_choice: BTreeMap::from([('A', 1)]),
18841 borda: BTreeMap::new(),
18842 winner: 'A',
18843 rankings: 1,
18844 unanimous_initial: true,
18845 deliberated: false,
18846 changed_votes: 0,
18847 unanimous_final: true,
18848 tie_break: None,
18849 judges: 1,
18850 present: 1,
18851 quorum: 1,
18852 met_quorum: true,
18853 uncontested: None,
18854 });
18855 state.status = RunStatus::Ready;
18856
18857 fold_run(&mut state, false, &crate::run::home())
18858 .await
18859 .expect("fold_run");
18860
18861 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
18862 assert!(
18863 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
18864 "the unmerged winner's branch survives"
18865 );
18866 assert!(
18867 !state.candidates[0].folded,
18868 "the winner is not marked folded"
18869 );
18870
18871 assert!(!wt_b.exists(), "the loser's worktree is removed");
18872 assert!(
18873 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
18874 "the loser's branch is removed"
18875 );
18876 assert!(state.candidates[1].folded, "the loser is marked folded");
18877 }
18878
18879 #[tokio::test]
18880 #[cfg(unix)]
18881 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
18882 use std::os::fd::AsRawFd as _;
18883 use std::time::Duration;
18884
18885 let tmp = tempfile::tempdir().unwrap();
18886 let repo = tmp.path().join("repo");
18887 std::fs::create_dir_all(&repo).unwrap();
18888 init_repo(&repo);
18889 let common = repo.join(".git");
18890 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18891 let home = tmp.path().join("home");
18892 let mut config = Config::default();
18893 config.coordination = crate::config::test_coordination();
18894 config.graph.worktree_root = Some(tmp.path().join("wt"));
18895 config.graph.timeout_resource_wait = 1;
18896 let mut state = RunState::new(
18897 repo.clone(),
18898 "main".into(),
18899 "deadbeef".into(),
18900 "task".into(),
18901 config,
18902 );
18903 let root = state.worktree_root();
18904 let candidate = root.join("cand-A");
18905 let reviewer = root.join("review-1");
18906 let branch = state.branch_for('A');
18907 git::worktree_add_branch(&repo, &candidate, &branch, "main")
18908 .await
18909 .unwrap();
18910 git::worktree_add_detached(&repo, &reviewer, "main")
18911 .await
18912 .unwrap();
18913 git::acquire_worktree_config(&repo).await.unwrap();
18914 state.enabled_worktree_config = true;
18915 state.status = RunStatus::Ready;
18916 state.candidates.push(Candidate {
18917 index: 0,
18918 label: 'A',
18919 agent: "implement".into(),
18920 branch: branch.clone(),
18921 worktree: candidate.clone(),
18922 summary: String::new(),
18923 stat: String::new(),
18924 files: 0,
18925 commits: 0,
18926 empty: false,
18927 failed: None,
18928 verified_noop: None,
18929 duration_ms: 0,
18930 folded: false,
18931 });
18932 state.save_under(&home).unwrap();
18933 let config_before = std::fs::read(common.join("config")).unwrap();
18934 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
18935 .await
18936 .unwrap();
18937 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18940 assert_eq!(
18941 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18942 0
18943 );
18944 let mut fold = Box::pin(fold_run(&mut state, true, &home));
18945 assert!(
18946 tokio::time::timeout(Duration::from_millis(200), &mut fold)
18947 .await
18948 .is_err()
18949 );
18950 assert!(candidate.exists());
18951 assert!(reviewer.exists());
18952 assert!(git::branch_exists(&repo, &branch).await.unwrap());
18953 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
18954 assert_eq!(
18955 git::git(&repo, &["worktree", "list", "--porcelain"])
18956 .await
18957 .unwrap(),
18958 worktrees_before
18959 );
18960 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
18961 .await
18962 .unwrap()
18963 .unwrap_err();
18964 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
18965 drop(fold);
18966 assert!(candidate.exists() && reviewer.exists());
18967 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
18968 assert_eq!(
18969 state.resource_block.as_ref().unwrap().kind,
18970 "repository coordination"
18971 );
18972 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
18973 drop(external);
18974 let mut fold = Box::pin(fold_run(&mut state, true, &home));
18975 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
18976 .await
18977 .unwrap()
18978 .unwrap();
18979 drop(fold);
18980 assert!(removed.contains(&branch));
18981 assert!(!root.exists());
18982 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
18983 assert!(state.candidates[0].folded);
18984 assert!(!state.enabled_worktree_config);
18985 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
18986 let enabled = git::git_raw(
18987 &repo,
18988 &["config", "--local", "--get", "extensions.worktreeConfig"],
18989 )
18990 .await
18991 .unwrap();
18992 assert!(!enabled.ok(), "the last config reference was released");
18993 }
18994
18995 #[tokio::test]
18998 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
18999 let tmp = tempfile::tempdir().expect("tempdir");
19000 let repo = tmp.path().join("repo");
19001 std::fs::create_dir_all(&repo).unwrap();
19002 init_repo(&repo);
19003 let home = tmp.path().join("home");
19004
19005 let mut config = Config::default();
19006 config.graph.worktree_root = Some(tmp.path().join("wt"));
19007 let mut state = RunState::new(
19008 repo.clone(),
19009 "main".to_owned(),
19010 "deadbeef".to_owned(),
19011 "task".to_owned(),
19012 config,
19013 );
19014 git::git(&repo, &["branch", "magi/x/A", "main"])
19016 .await
19017 .expect("branch");
19018 state.candidates = vec![Candidate {
19019 index: 0,
19020 label: 'A',
19021 agent: "alpha".to_owned(),
19022 branch: "magi/x/A".to_owned(),
19023 worktree: state.worktree_root().join("cand-A"),
19024 summary: String::new(),
19025 stat: String::new(),
19026 files: 0,
19027 commits: 0,
19028 empty: false,
19029 failed: None,
19030 verified_noop: None,
19031 duration_ms: 0,
19032 folded: true,
19033 }];
19034 state.released_to = Some("20260901-000000-new1".to_owned());
19035 state.released_branches = vec!["magi/x/A".to_owned()];
19036
19037 fold_run(&mut state, true, &home).await.expect("fold_run");
19038 state.status = RunStatus::Merged;
19039 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19040 state.merge = Some(crate::run::MergeOutcome {
19041 mode: MergeMode::Pr,
19042 ok: true,
19043 detail: "confirmed merged".into(),
19044 empty: false,
19045 });
19046 fold_run(&mut state, true, &home)
19047 .await
19048 .expect("merged fold");
19049
19050 assert!(
19051 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19052 "the handed-over branch survives a fold"
19053 );
19054 }
19055
19056 #[tokio::test]
19060 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19061 let tmp = tempfile::tempdir().expect("tempdir");
19062 let repo = tmp.path().join("repo");
19063 std::fs::create_dir_all(&repo).unwrap();
19064 init_repo(&repo);
19065 let run = |args: &[&str]| {
19066 let out = std::process::Command::new("git")
19067 .quiet()
19068 .args(args)
19069 .current_dir(&repo)
19070 .output()
19071 .expect("spawn git");
19072 assert!(out.status.success(), "git {args:?}");
19073 };
19074 run(&["branch", "magi/x/A"]);
19075 run(&["checkout", "-q", "-b", "magi/x/B"]);
19076 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19077 run(&["add", "-A"]);
19078 run(&["commit", "-q", "-m", "work"]);
19079 run(&["checkout", "-q", "main"]);
19080
19081 let probe = RunState::new(
19085 repo.clone(),
19086 "main".to_owned(),
19087 "deadbeef".to_owned(),
19088 "task".to_owned(),
19089 Config::default(),
19090 );
19091 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19092 let bare = tmp.path().join("origin.git");
19093 let out = std::process::Command::new("git")
19094 .quiet()
19095 .args(["init", "-q", "--bare"])
19096 .arg(&bare)
19097 .output()
19098 .expect("spawn git");
19099 assert!(out.status.success(), "git init --bare");
19100 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19101 run(&["push", "-q", "origin", "main"]);
19102
19103 let mut state = RunState::new(
19104 repo.clone(),
19105 "main".to_owned(),
19106 "deadbeef".to_owned(),
19107 "task".to_owned(),
19108 Config::default(),
19109 );
19110 state.seeds = vec![refs::Seed {
19111 token: "magi/27b2/A".to_owned(),
19112 kind: refs::SeedKind::Unresolved,
19113 sha: String::new(),
19114 branch: true,
19115 detail: "no branch or commit named magi/27b2/A".to_owned(),
19116 }];
19117
19118 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19119 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19120 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19121 let detail = empty_candidate_detail(&state, "main");
19122 assert!(detail.starts_with("empty candidate"), "{detail}");
19123 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19124 assert!(detail.contains("--start-from"), "{detail}");
19125 }
19126
19127 #[tokio::test]
19136 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19137 let tmp = tempfile::tempdir().expect("tempdir");
19138 let repo = tmp.path().join("repo");
19139 std::fs::create_dir_all(&repo).unwrap();
19140 init_repo(&repo);
19141
19142 let mut config = Config::default();
19143 config.merge.mode = MergeMode::Local;
19144
19145 let mut state = RunState::new(
19146 repo.clone(),
19147 "main".to_owned(),
19148 "deadbeef".to_owned(),
19149 "task".to_owned(),
19150 config,
19151 );
19152 state.candidates = vec![Candidate {
19153 index: 0,
19154 label: 'A',
19155 agent: "alpha".to_owned(),
19156 branch: "does-not-exist".to_owned(),
19157 worktree: repo.clone(),
19158 summary: String::new(),
19159 stat: String::new(),
19160 files: 0,
19161 commits: 0,
19162 empty: false,
19163 failed: None,
19164 verified_noop: None,
19165 duration_ms: 0,
19166 folded: false,
19167 }];
19168 state.tally = Some(Tally {
19169 first_choice: BTreeMap::from([('A', 1)]),
19170 borda: BTreeMap::new(),
19171 winner: 'A',
19172 rankings: 1,
19173 unanimous_initial: true,
19174 deliberated: false,
19175 changed_votes: 0,
19176 unanimous_final: true,
19177 tie_break: None,
19178 judges: 0,
19179 present: 0,
19180 quorum: 0,
19181 met_quorum: true,
19182 uncontested: Some("only candidate A produced a change".to_owned()),
19183 });
19184 state.reviews = vec![ReviewRound {
19185 round: 1,
19186 head: "deadbeef".to_owned(),
19187 verified_head: None,
19188 verified_at: None,
19189 reviews: Vec::new(),
19190 e2e: Vec::new(),
19191 fix: None,
19192 blocking: 0,
19193 answered: 0,
19194 expected: 0,
19195 clean: true,
19196 verify_retried: false,
19197 e2e_deferred: false,
19198 e2e_defer_reason: None,
19199 progressed: false,
19200 vote_split: false,
19201 reconsideration: Vec::new(),
19202 verdict: None,
19203 }];
19204 state.gate = vec![CommandOutcome {
19205 command: "test".to_owned(),
19206 code: Some(0),
19207 output_tail: String::new(),
19208 duration_ms: 0,
19209 resource_blocked: false,
19210 }];
19211 state.gate_ran = true;
19212 state.status = RunStatus::Ready;
19217 state.merge = Some(MergeOutcome {
19218 mode: MergeMode::Local,
19219 ok: false,
19220 detail: "already concluded".to_owned(),
19221 empty: false,
19222 });
19223
19224 let mut runner = Runner {
19225 state,
19226 roles: ResolvedRoles {
19227 implementers: Vec::new(),
19228 judges: Vec::new(),
19229 reviewers: Vec::new(),
19230 fixer: None,
19231 conductor: conductor(),
19232 implementer_roster: Vec::new(),
19233 judge_roster: Vec::new(),
19234 reviewer_roster: Vec::new(),
19235 },
19236 sem: Arc::new(Semaphore::new(1)),
19237 pause: Pause::new(),
19238 interrupt: Pause::new(),
19239 };
19240
19241 runner.merge().await.expect("merge");
19242
19243 assert_eq!(
19244 runner.state.status,
19245 RunStatus::Ready,
19246 "a concluded run's status must not change on reentry"
19247 );
19248 assert_eq!(
19249 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19250 Some("already concluded"),
19251 "merge must not run again once the node already recorded an outcome"
19252 );
19253 }
19254
19255 #[tokio::test]
19264 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19265 let tmp = tempfile::tempdir().expect("tempdir");
19266 let repo = tmp.path().join("repo");
19267 std::fs::create_dir_all(&repo).unwrap();
19268 init_repo(&repo);
19269
19270 let mut config = Config::default();
19271 config.merge.mode = MergeMode::Local;
19272
19273 let mut state = RunState::new(
19274 repo.clone(),
19275 "main".to_owned(),
19276 "deadbeef".to_owned(),
19277 "task".to_owned(),
19278 config,
19279 );
19280 state.candidates = vec![Candidate {
19281 index: 0,
19282 label: 'A',
19283 agent: "alpha".to_owned(),
19284 branch: "does-not-exist".to_owned(),
19285 worktree: repo.clone(),
19286 summary: String::new(),
19287 stat: String::new(),
19288 files: 0,
19289 commits: 0,
19290 empty: false,
19291 failed: None,
19292 verified_noop: None,
19293 duration_ms: 0,
19294 folded: false,
19295 }];
19296 state.tally = Some(Tally {
19297 first_choice: BTreeMap::from([('A', 1)]),
19298 borda: BTreeMap::new(),
19299 winner: 'A',
19300 rankings: 1,
19301 unanimous_initial: true,
19302 deliberated: false,
19303 changed_votes: 0,
19304 unanimous_final: true,
19305 tie_break: None,
19306 judges: 0,
19307 present: 0,
19308 quorum: 0,
19309 met_quorum: true,
19310 uncontested: Some("only candidate A produced a change".to_owned()),
19311 });
19312 state.reviews = vec![ReviewRound {
19313 round: 1,
19314 head: "deadbeef".to_owned(),
19315 verified_head: None,
19316 verified_at: None,
19317 reviews: Vec::new(),
19318 e2e: Vec::new(),
19319 fix: None,
19320 blocking: 0,
19321 answered: 0,
19322 expected: 0,
19323 clean: true,
19324 verify_retried: false,
19325 e2e_deferred: false,
19326 e2e_defer_reason: None,
19327 progressed: false,
19328 vote_split: false,
19329 reconsideration: Vec::new(),
19330 verdict: None,
19331 }];
19332 state.gate = Vec::new();
19334 state.gate_ran = false;
19335 state.status = RunStatus::Gating;
19336
19337 let mut runner = Runner {
19338 state,
19339 roles: ResolvedRoles {
19340 implementers: Vec::new(),
19341 judges: Vec::new(),
19342 reviewers: Vec::new(),
19343 fixer: None,
19344 conductor: conductor(),
19345 implementer_roster: Vec::new(),
19346 judge_roster: Vec::new(),
19347 reviewer_roster: Vec::new(),
19348 },
19349 sem: Arc::new(Semaphore::new(1)),
19350 pause: Pause::new(),
19351 interrupt: Pause::new(),
19352 };
19353
19354 runner.merge().await.expect("merge");
19355
19356 assert!(
19357 runner.state.merge.is_none(),
19358 "an empty gate must never be read as a passing one: {:?}",
19359 runner.state.merge
19360 );
19361 }
19362
19363 #[tokio::test]
19370 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19371 let tmp = tempfile::tempdir().expect("tempdir");
19372 let repo = tmp.path().join("repo");
19373 std::fs::create_dir_all(&repo).unwrap();
19374 init_repo(&repo);
19375
19376 let config = Config::default();
19378
19379 let mut state = RunState::new(
19380 repo.clone(),
19381 "main".to_owned(),
19382 "deadbeef".to_owned(),
19383 "task".to_owned(),
19384 config,
19385 );
19386 state.candidates = vec![Candidate {
19387 index: 0,
19388 label: 'A',
19389 agent: "alpha".to_owned(),
19390 branch: "does-not-exist".to_owned(),
19391 worktree: repo.clone(),
19392 summary: String::new(),
19393 stat: String::new(),
19394 files: 0,
19395 commits: 0,
19396 empty: false,
19397 failed: None,
19398 verified_noop: None,
19399 duration_ms: 0,
19400 folded: false,
19401 }];
19402 state.tally = Some(Tally {
19403 first_choice: BTreeMap::from([('A', 1)]),
19404 borda: BTreeMap::new(),
19405 winner: 'A',
19406 rankings: 1,
19407 unanimous_initial: true,
19408 deliberated: false,
19409 changed_votes: 0,
19410 unanimous_final: true,
19411 tie_break: None,
19412 judges: 0,
19413 present: 0,
19414 quorum: 0,
19415 met_quorum: true,
19416 uncontested: Some("only candidate A produced a change".to_owned()),
19417 });
19418 state.reviews = vec![ReviewRound {
19419 round: 1,
19420 head: "deadbeef".to_owned(),
19421 verified_head: None,
19422 verified_at: None,
19423 reviews: Vec::new(),
19424 e2e: Vec::new(),
19425 fix: None,
19426 blocking: 0,
19427 answered: 0,
19428 expected: 0,
19429 clean: true,
19430 verify_retried: false,
19431 e2e_deferred: false,
19432 e2e_defer_reason: None,
19433 progressed: false,
19434 vote_split: false,
19435 reconsideration: Vec::new(),
19436 verdict: None,
19437 }];
19438
19439 let mut runner = Runner {
19440 state,
19441 roles: ResolvedRoles {
19442 implementers: Vec::new(),
19443 judges: Vec::new(),
19444 reviewers: Vec::new(),
19445 fixer: None,
19446 conductor: conductor(),
19447 implementer_roster: Vec::new(),
19448 judge_roster: Vec::new(),
19449 reviewer_roster: Vec::new(),
19450 },
19451 sem: Arc::new(Semaphore::new(1)),
19452 pause: Pause::new(),
19453 interrupt: Pause::new(),
19454 };
19455
19456 runner.gate().await.expect("gate");
19457 assert!(
19458 runner.state.gate_ran,
19459 "zero configured commands is still a real attempt, not an unrun gate"
19460 );
19461 assert!(runner.state.gate.is_empty());
19462 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19463 assert_ne!(
19464 runner.state.status,
19465 RunStatus::Blocked,
19466 "a gate with nothing to check must not read as failed"
19467 );
19468
19469 runner.merge().await.expect("merge");
19470 assert_eq!(
19471 runner.state.status,
19472 RunStatus::Ready,
19473 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19474 );
19475 }
19476
19477 #[tokio::test]
19488 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19489 crate::run::pin_test_home();
19490 let home = crate::run::home();
19491
19492 let tmp = tempfile::tempdir().expect("tempdir");
19493 let repo = tmp.path().join("repo");
19494 std::fs::create_dir_all(&repo).unwrap();
19495 init_repo(&repo);
19496 let cache_dir = tmp.path().join("target");
19499
19500 let mut config = Config::default();
19501 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19502 config.graph.timeout_verify = Some(2);
19506 config.graph.timeout_resource_wait = 0;
19507
19508 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19509 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19510 .expect("no io error acquiring directly")
19511 {
19512 crate::cache::AcquireOutcome::Acquired(g) => g,
19513 crate::cache::AcquireOutcome::Busy(b) => {
19514 panic!("expected the direct acquire to win the lease first: {b:?}")
19515 }
19516 };
19517
19518 let mut state = RunState::new(
19519 repo.clone(),
19520 "main".to_owned(),
19521 "deadbeef".to_owned(),
19522 "task".to_owned(),
19523 config,
19524 );
19525 state.candidates = vec![Candidate {
19526 index: 0,
19527 label: 'A',
19528 agent: "alpha".to_owned(),
19529 branch: "does-not-exist".to_owned(),
19530 worktree: repo.clone(),
19531 summary: String::new(),
19532 stat: String::new(),
19533 files: 0,
19534 commits: 0,
19535 empty: false,
19536 failed: None,
19537 verified_noop: None,
19538 duration_ms: 0,
19539 folded: false,
19540 }];
19541 state.tally = Some(Tally {
19542 first_choice: BTreeMap::from([('A', 1)]),
19543 borda: BTreeMap::new(),
19544 winner: 'A',
19545 rankings: 1,
19546 unanimous_initial: true,
19547 deliberated: false,
19548 changed_votes: 0,
19549 unanimous_final: true,
19550 tie_break: None,
19551 judges: 0,
19552 present: 0,
19553 quorum: 0,
19554 met_quorum: true,
19555 uncontested: Some("only candidate A produced a change".to_owned()),
19556 });
19557 state.reviews = vec![ReviewRound {
19558 round: 1,
19559 head: "deadbeef".to_owned(),
19560 verified_head: None,
19561 verified_at: None,
19562 reviews: Vec::new(),
19563 e2e: Vec::new(),
19564 fix: None,
19565 blocking: 0,
19566 answered: 0,
19567 expected: 0,
19568 clean: true,
19569 verify_retried: false,
19570 e2e_deferred: false,
19571 e2e_defer_reason: None,
19572 progressed: false,
19573 vote_split: false,
19574 reconsideration: Vec::new(),
19575 verdict: None,
19576 }];
19577
19578 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19579 state.reviews[0].e2e = vec![CommandOutcome {
19580 command: state.config.verify.e2e[0].clone(),
19581 code: Some(0),
19582 output_tail: String::new(),
19583 duration_ms: 0,
19584 resource_blocked: false,
19585 }];
19586 let mut runner = Runner {
19587 state,
19588 roles: ResolvedRoles {
19589 implementers: Vec::new(),
19590 judges: Vec::new(),
19591 reviewers: Vec::new(),
19592 fixer: None,
19593 conductor: conductor(),
19594 implementer_roster: Vec::new(),
19595 judge_roster: Vec::new(),
19596 reviewer_roster: Vec::new(),
19597 },
19598 sem: Arc::new(Semaphore::new(1)),
19599 pause: Pause::new(),
19600 interrupt: Pause::new(),
19601 };
19602
19603 runner.gate().await.expect("gate");
19604 assert!(runner.state.resource_block.is_none());
19605 assert!(
19606 !runner
19607 .state
19608 .events
19609 .iter()
19610 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19611 "a gate with nothing to run must never request the held cache lease"
19612 );
19613 assert!(
19614 runner.state.gate_ran,
19615 "zero commands is still a real, immediate attempt"
19616 );
19617 assert!(runner.state.gate.is_empty());
19618 assert_ne!(
19619 runner.state.status,
19620 RunStatus::Blocked,
19621 "must not read as resource-blocked on a lease it never asked for"
19622 );
19623 }
19624
19625 #[cfg(unix)]
19628 #[tokio::test]
19629 async fn verification_is_standalone_inside_a_live_worker() {
19630 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19631 if std::env::var_os(CHILD).is_none() {
19632 let home = tempfile::tempdir().unwrap();
19633 let generation = crate::worker::pin_current(home.path()).unwrap();
19634 let output = std::process::Command::new(std::env::current_exe().unwrap())
19635 .args([
19636 "--exact",
19637 "graph::tests::verification_is_standalone_inside_a_live_worker",
19638 "--nocapture",
19639 ])
19640 .env(CHILD, "1")
19641 .env("MAGI_HOME", home.path())
19642 .env("MAGI_WORKER_HOME", home.path())
19643 .env("MAGI_WORKER_ID", "abcdef")
19644 .env("MAGI_WORKER_GENERATION", &generation.id)
19645 .env("MAGI_WORKER_EXE", &generation.executable)
19646 .env("MAGI_RUN", "verification-parent")
19647 .env("MAGI_NODE", "gate")
19648 .env("MAGI_SEAT", "worker-fixture")
19649 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19650 .env("MAGI_NO_AUTOUPDATE", "0")
19651 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19652 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19653 .env(
19654 "VERIFY_EXPECT_TOOLCHAIN",
19655 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19656 )
19657 .env(
19658 "VERIFY_EXPECT_CACHE",
19659 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19660 )
19661 .output()
19662 .unwrap();
19663 assert!(
19664 output.status.success()
19665 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19666 "{}\n{}",
19667 String::from_utf8_lossy(&output.stdout),
19668 String::from_utf8_lossy(&output.stderr)
19669 );
19670 return;
19671 }
19672 let home = crate::run::home();
19673 let directory = home.join("workers/jobs/abcdef");
19674 std::fs::create_dir_all(&directory).unwrap();
19675 let job = crate::worker::Job {
19676 id: "abcdef".into(),
19677 sequence: 1,
19678 kind: "task".into(),
19679 key: "task:verification".into(),
19680 payload: serde_json::json!({}),
19681 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19682 version: Some(env!("CARGO_PKG_VERSION").into()),
19683 status: crate::worker::Status::Running,
19684 result: None,
19685 pid: Some(std::process::id()),
19686 created_at: 0,
19687 finished_at: None,
19688 };
19689 std::fs::write(
19690 directory.join("job.json"),
19691 serde_json::to_vec(&job).unwrap(),
19692 )
19693 .unwrap();
19694 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19695 let assert_worker = || {
19696 assert_eq!(
19697 crate::worker::current_job(&home).unwrap().unwrap().id,
19698 job.id
19699 );
19700 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19701 assert_eq!(
19702 crate::worker::helper_executable(&home).unwrap(),
19703 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19704 );
19705 assert!(
19706 crate::updater::Checker::new(&crate::config::Update {
19707 mode: crate::config::UpdateMode::Notify,
19708 interval: None,
19709 auto_restart: false,
19710 })
19711 .is_none(),
19712 "the driver remains pinned, even while tests are standalone"
19713 );
19714 };
19715 assert_worker();
19716 let repo = home.join("repo");
19717 std::fs::create_dir_all(&repo).unwrap();
19718 init_repo(&repo);
19719 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19722 test -z "${MAGI_WORKER_GENERATION+x}" &&
19723 test -z "${MAGI_WORKER_HOME+x}" &&
19724 test -z "${MAGI_WORKER_EXE+x}" &&
19725 test "$MAGI_HOME/repo" -ef . &&
19726 test "$MAGI_RUN" = verification-parent &&
19727 test "$MAGI_NODE" = gate &&
19728 test "$MAGI_SEAT" = worker-fixture &&
19729 test -n "$MAGI_PROCESS_OWNER" &&
19730 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19731 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19732 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19733 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19734 assert!(
19736 !std::process::Command::new("sh")
19737 .args(["-c", probe])
19738 .current_dir(&repo)
19739 .status()
19740 .unwrap()
19741 .success()
19742 );
19743 let unrelated_home = tempfile::tempdir().unwrap();
19744 assert!(
19745 crate::worker::current_job(unrelated_home.path()).is_err(),
19746 "an inherited worker id cannot be resolved in a unit fixture's home"
19747 );
19748 let mut runner = runner_at(RunStatus::Gating);
19749 runner.state.repo = repo.clone();
19750 runner.state.config.disk.min_free_bytes = 0;
19751 runner.state.config.coordination = crate::config::test_coordination();
19752 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19753 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19754 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19755 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19756 let winner = Candidate {
19757 index: 0,
19758 label: 'A',
19759 agent: "fixture".into(),
19760 branch: "main".into(),
19761 worktree: repo.clone(),
19762 summary: String::new(),
19763 stat: String::new(),
19764 files: 0,
19765 commits: 0,
19766 empty: false,
19767 failed: None,
19768 verified_noop: None,
19769 duration_ms: 0,
19770 folded: false,
19771 };
19772 let (e2e, retried, pids) = run_e2e_with_retry(
19773 &mut runner.state,
19774 &["sh".into(), "-c".into()],
19775 &[format!("{probe} && printf initial")],
19776 &repo,
19777 Duration::from_secs(10),
19778 "initial",
19779 )
19780 .await;
19781 assert!(!retried && pids.is_empty());
19782 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
19783 assert_eq!(e2e[0].output_tail, "initial");
19784 let gate = runner.run_gate(&winner).await.unwrap();
19785 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
19786 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
19789 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
19790 passed: false,
19791 outcomes: Vec::new(),
19792 });
19793 runner.state.save().unwrap();
19794 let mut resumed = RunState::load(&runner.state.id).unwrap();
19795 let wait = crate::repo_lock::WaitPolicy::default();
19796 assert!(matches!(
19797 check_gate_repair_e2e(&mut resumed, &winner, &wait)
19798 .await
19799 .unwrap(),
19800 GateFix::Retry
19801 ));
19802 let saved = RunState::load(&resumed.id).unwrap();
19803 let repair = saved.gate_repair_e2e.unwrap();
19804 assert!(repair.passed);
19805 assert_eq!(repair.outcomes.len(), 1);
19806 assert!(repair.outcomes[0].ok());
19807 let landing = run_landing_gate(&mut resumed, &winner, &wait)
19808 .await
19809 .unwrap();
19810 assert_eq!(landing.len(), 1);
19811 assert!(landing[0].ok(), "{landing:?}");
19812 assert_eq!(
19813 std::fs::read_to_string(repo.join("phases")).unwrap(),
19814 "pre\ngate\ne2e\npre\ngate\n"
19815 );
19816 assert_worker();
19817 }
19818
19819 #[cfg(unix)]
19820 #[tokio::test]
19821 async fn verification_keeps_the_panic_and_the_full_output() {
19822 crate::run::pin_test_home();
19823 let tmp = tempfile::tempdir().unwrap();
19824 let mut config = Config::default();
19825 config.disk.min_free_bytes = 0;
19826 let mut state = RunState::new(
19827 tmp.path().to_owned(),
19828 "main".into(),
19829 "base".into(),
19830 "verification evidence".into(),
19831 config,
19832 );
19833 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
19834 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
19835 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
19836 let mut names = Vec::new();
19837 for _ in 0..2 {
19838 let (outcomes, _) = run_commands(
19839 &mut state,
19840 "gate",
19841 "gate",
19842 0,
19843 &["sh".into(), "-c".into()],
19844 &[script.into()],
19845 tmp.path(),
19846 Duration::from_secs(60),
19847 )
19848 .await;
19849 let tail = &outcomes[0].output_tail;
19850 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
19851 assert!(tail.contains("boom assertion"));
19852 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
19853 let line = tail.lines().next().unwrap();
19854 names.push(line.to_owned());
19855 }
19856 assert_ne!(names[0], names[1], "the second run must not overwrite");
19857 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
19858 assert!(full.len() > 500_000 && full.contains("boom assertion"));
19859 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
19860
19861 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
19863 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
19864 let (outcomes, _) = run_commands(
19865 &mut state,
19866 "gate",
19867 "gate",
19868 0,
19869 &["sh".into(), "-c".into()],
19870 &["echo failure-output; exit 7".into()],
19871 tmp.path(),
19872 Duration::from_secs(10),
19873 )
19874 .await;
19875 assert_eq!(outcomes[0].code, Some(7));
19876 assert!(state.events.iter().any(|e| {
19877 e.node == "gate" && e.message.contains("could not save the full gate output:")
19878 }));
19879 }
19880
19881 #[cfg(unix)]
19882 #[tokio::test]
19883 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
19884 crate::run::pin_test_home();
19885 let tmp = tempfile::tempdir().unwrap();
19886 let mut config = Config::default();
19887 config.disk.min_free_bytes = 0;
19888 let mut state = RunState::new(
19889 tmp.path().to_owned(),
19890 "main".into(),
19891 "base".into(),
19892 "verification evidence".into(),
19893 config,
19894 );
19895 let (outcomes, pids) = run_commands(
19896 &mut state,
19897 "gate",
19898 "gate",
19899 1,
19900 &["sh".into(), "-c".into()],
19901 &[
19902 "printf before-timeout; exec sleep 30".into(),
19903 "kill -TERM $$".into(),
19904 "exit 7".into(),
19905 "printf success".into(),
19906 ],
19907 tmp.path(),
19908 Duration::from_secs(1),
19909 )
19910 .await;
19911 wait_for_timed_out_children_to_die(&pids).await;
19912 assert_eq!(
19913 pids.len(),
19914 1,
19915 "unleased timeout retains its death-check PID"
19916 );
19917 assert_eq!(outcomes.len(), 4);
19918 assert_eq!(outcomes[0].code, None);
19919 assert!(outcomes[0].output_tail.contains("before-timeout"));
19920 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
19921 assert_eq!(outcomes[1].code, None);
19922 assert!(outcomes[1].output_tail.contains("command terminated:"));
19923 assert!(!outcomes[1].output_tail.contains("timed out"));
19924 assert_eq!(outcomes[2].code, Some(7));
19925 assert!(!outcomes[2].output_tail.contains("timed out"));
19926 assert_eq!(outcomes[3].code, Some(0));
19927 assert_eq!(outcomes[3].output_tail, "success");
19928 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
19929 }
19930
19931 #[tokio::test]
19941 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
19942 crate::run::pin_test_home();
19943
19944 let tmp = tempfile::tempdir().expect("tempdir");
19945 let repo = tmp.path().join("repo");
19946 std::fs::create_dir_all(&repo).unwrap();
19947 init_repo(&repo);
19948
19949 let mut config = Config::default();
19950 config.verify.gate = vec![
19951 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
19952 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
19953 .to_owned(),
19954 ];
19955
19956 let mut state = RunState::new(
19957 repo.clone(),
19958 "main".to_owned(),
19959 "deadbeef".to_owned(),
19960 "task".to_owned(),
19961 config,
19962 );
19963 let run_id = state.id.clone();
19964 state.candidates = vec![Candidate {
19965 index: 0,
19966 label: 'A',
19967 agent: "alpha".to_owned(),
19968 branch: "does-not-exist".to_owned(),
19969 worktree: repo.clone(),
19970 summary: String::new(),
19971 stat: String::new(),
19972 files: 0,
19973 commits: 0,
19974 empty: false,
19975 failed: None,
19976 verified_noop: None,
19977 duration_ms: 0,
19978 folded: false,
19979 }];
19980 state.tally = Some(Tally {
19981 first_choice: BTreeMap::from([('A', 1)]),
19982 borda: BTreeMap::new(),
19983 winner: 'A',
19984 rankings: 1,
19985 unanimous_initial: true,
19986 deliberated: false,
19987 changed_votes: 0,
19988 unanimous_final: true,
19989 tie_break: None,
19990 judges: 0,
19991 present: 0,
19992 quorum: 0,
19993 met_quorum: true,
19994 uncontested: Some("only candidate A produced a change".to_owned()),
19995 });
19996 state.reviews = vec![ReviewRound {
19997 round: 1,
19998 head: "deadbeef".to_owned(),
19999 verified_head: None,
20000 verified_at: None,
20001 reviews: Vec::new(),
20002 e2e: Vec::new(),
20003 fix: None,
20004 blocking: 0,
20005 answered: 0,
20006 expected: 0,
20007 clean: true,
20008 verify_retried: false,
20009 e2e_deferred: false,
20010 e2e_defer_reason: None,
20011 progressed: false,
20012 vote_split: false,
20013 reconsideration: Vec::new(),
20014 verdict: None,
20015 }];
20016
20017 let mut runner = Runner {
20018 state,
20019 roles: ResolvedRoles {
20020 implementers: Vec::new(),
20021 judges: Vec::new(),
20022 reviewers: Vec::new(),
20023 fixer: None,
20024 conductor: conductor(),
20025 implementer_roster: Vec::new(),
20026 judge_roster: Vec::new(),
20027 reviewer_roster: Vec::new(),
20028 },
20029 sem: Arc::new(Semaphore::new(1)),
20030 pause: Pause::new(),
20031 interrupt: Pause::new(),
20032 };
20033
20034 let started_marker = repo.join("started.marker");
20035 let release_marker = repo.join("release.marker");
20036 let poller = tokio::spawn(async move {
20037 for _ in 0..100 {
20042 if started_marker.exists()
20043 && let Ok(s) = crate::run::RunState::load(&run_id)
20044 && let Some(a) = s.active.get("gate")
20045 {
20046 std::fs::write(&release_marker, b"go").expect("release marker");
20047 return Some(a.clone());
20048 }
20049 tokio::time::sleep(Duration::from_millis(50)).await;
20050 }
20051 None
20052 });
20053
20054 runner.gate().await.expect("gate");
20055 let captured = poller.await.expect("poller task");
20056 let captured = captured.expect(
20057 "the poller never saw a `gate` task entry in run.json while the command was \
20058 still blocked on its own release marker",
20059 );
20060
20061 assert_eq!(captured.task.as_deref(), Some("gate"));
20062 assert_eq!(captured.node, "gate");
20063 assert_eq!(captured.index, Some(1));
20064 assert_eq!(captured.total, Some(1));
20065 assert!(
20066 captured
20067 .command
20068 .as_deref()
20069 .is_some_and(|c| c.contains("started.marker")),
20070 "{captured:?}"
20071 );
20072
20073 assert!(
20074 runner.state.active.is_empty(),
20075 "the entry must be cleared once the command actually finished: {:?}",
20076 runner.state.active
20077 );
20078 assert!(runner.state.gate_ran);
20079 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20080 }
20081
20082 #[tokio::test]
20085 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20086 use crate::verdict::{Finding, ReviewVote, Severity};
20087 crate::run::pin_test_home();
20088 let tmp = tempfile::tempdir().expect("tempdir");
20089 let repo = tmp.path().join("repo");
20090 std::fs::create_dir_all(&repo).unwrap();
20091 init_repo(&repo);
20092
20093 for (severity, vote, expect) in [
20094 (Severity::Major, ReviewVote::Reject, true),
20095 (Severity::Minor, ReviewVote::Reject, false),
20096 (Severity::Major, ReviewVote::Approve, false),
20097 ] {
20098 let mut round = review_round(false, 1, 1, 1, false, true);
20099 round.reviews = vec![ReviewRecord {
20100 reviewer: 1,
20101 agent: "alpha".to_owned(),
20102 summary: String::new(),
20103 findings: vec![Finding {
20104 id: "R1-1-1".to_owned(),
20105 severity,
20106 file: None,
20107 line: None,
20108 title: "t".to_owned(),
20109 detail: String::new(),
20110 }],
20111 vote: Some(vote),
20112 failed: None,
20113 duration_ms: 0,
20114 attempts: 0,
20115 }];
20116 let mut state = RunState::new(
20117 repo.clone(),
20118 "main".to_owned(),
20119 "deadbeef".to_owned(),
20120 "task".to_owned(),
20121 Config::default(),
20122 );
20123 state.reviews = vec![round];
20124 let mut runner = Runner {
20125 state,
20126 roles: ResolvedRoles {
20127 implementers: Vec::new(),
20128 judges: Vec::new(),
20129 reviewers: Vec::new(),
20130 fixer: None,
20131 conductor: conductor(),
20132 implementer_roster: Vec::new(),
20133 judge_roster: Vec::new(),
20134 reviewer_roster: Vec::new(),
20135 },
20136 sem: Arc::new(Semaphore::new(1)),
20137 pause: Pause::new(),
20138 interrupt: Pause::new(),
20139 };
20140 let shell = runner.state.config.shell();
20141 runner
20142 .stop_reviewing("round budget spent", &shell, &repo)
20143 .await
20144 .expect("stop_reviewing");
20145 assert_eq!(runner.state.status, RunStatus::Gating);
20146 assert_eq!(
20147 runner.state.contested_handoff.is_some(),
20148 expect,
20149 "{severity:?} + {vote:?}"
20150 );
20151 }
20152 }
20153
20154 #[tokio::test]
20167 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20168 crate::run::pin_test_home();
20169 let home = crate::run::home();
20170
20171 let tmp = tempfile::tempdir().expect("tempdir");
20172 let repo = tmp.path().join("repo");
20173 std::fs::create_dir_all(&repo).unwrap();
20174 init_repo(&repo);
20175 let head = crate::git::rev_parse(&repo, "HEAD")
20176 .await
20177 .expect("rev-parse");
20178 let cache_dir = tmp.path().join("target");
20181
20182 let mut config = Config::default();
20183 config.verify.e2e = vec![format!(
20184 "CARGO_TARGET_DIR='{}' test -f README.md",
20185 cache_dir.display()
20186 )];
20187 config.graph.review_rounds = 1;
20188 config.graph.timeout_verify = Some(2);
20191 config.graph.timeout_resource_wait = 2;
20192
20193 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20194 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20195 .expect("no io error acquiring directly")
20196 {
20197 crate::cache::AcquireOutcome::Acquired(g) => g,
20198 crate::cache::AcquireOutcome::Busy(b) => {
20199 panic!("expected the direct acquire to win the lease first: {b:?}")
20200 }
20201 };
20202
20203 let mut state = RunState::new(
20204 repo.clone(),
20205 "main".to_owned(),
20206 head.clone(),
20207 "task".to_owned(),
20208 config,
20209 );
20210 state.candidates = vec![Candidate {
20211 index: 0,
20212 label: 'A',
20213 agent: "alpha".to_owned(),
20214 branch: "does-not-exist".to_owned(),
20215 worktree: repo.clone(),
20216 summary: String::new(),
20217 stat: String::new(),
20218 files: 0,
20219 commits: 0,
20220 empty: false,
20221 failed: None,
20222 verified_noop: None,
20223 duration_ms: 0,
20224 folded: false,
20225 }];
20226 state.tally = Some(Tally {
20227 first_choice: BTreeMap::from([('A', 1)]),
20228 borda: BTreeMap::new(),
20229 winner: 'A',
20230 rankings: 1,
20231 unanimous_initial: true,
20232 deliberated: false,
20233 changed_votes: 0,
20234 unanimous_final: true,
20235 tie_break: None,
20236 judges: 0,
20237 present: 0,
20238 quorum: 0,
20239 met_quorum: true,
20240 uncontested: Some("only candidate A produced a change".to_owned()),
20241 });
20242 state.reviews = vec![ReviewRound {
20246 round: 1,
20247 head: head.clone(),
20248 verified_head: None,
20249 verified_at: None,
20250 reviews: Vec::new(),
20251 e2e: Vec::new(),
20252 fix: None,
20253 blocking: 1,
20254 answered: 1,
20255 expected: 1,
20256 clean: false,
20257 verify_retried: false,
20258 e2e_deferred: true,
20259 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20260 progressed: false,
20261 vote_split: false,
20262 reconsideration: Vec::new(),
20263 verdict: None,
20264 }];
20265
20266 let mut runner = Runner {
20267 state,
20268 roles: ResolvedRoles {
20269 implementers: Vec::new(),
20270 judges: Vec::new(),
20271 reviewers: Vec::new(),
20272 fixer: None,
20273 conductor: conductor(),
20274 implementer_roster: Vec::new(),
20275 judge_roster: Vec::new(),
20276 reviewer_roster: Vec::new(),
20277 },
20278 sem: Arc::new(Semaphore::new(1)),
20279 pause: Pause::new(),
20280 interrupt: Pause::new(),
20281 };
20282
20283 let shell = runner.state.config.shell();
20284 runner
20285 .stop_reviewing("round budget spent", &shell, &repo)
20286 .await
20287 .expect("stop_reviewing");
20288
20289 let last = runner.state.reviews.last().expect("round record");
20290 assert_eq!(
20291 last.e2e_status(),
20292 E2eStatus::ResourceBlocked,
20293 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20294 failed: {last:?}"
20295 );
20296 assert_eq!(
20297 last.verified_head.as_deref(),
20298 Some(head.as_str()),
20299 "which commit this attempt targeted is known even though nothing finished checking \
20300 it"
20301 );
20302 let first_attempt_at = last
20303 .verified_at
20304 .expect("when this attempt ran is known too");
20305 assert_ne!(
20306 runner.state.status,
20307 RunStatus::Blocked,
20308 "contention is evidence about the machine, not the patch — it must not settle the \
20309 run as blocked: {:?}",
20310 runner.state.status
20311 );
20312 assert!(
20313 !runner
20314 .state
20315 .events
20316 .iter()
20317 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20318 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20319 runner.state.events
20320 );
20321
20322 runner
20327 .stop_reviewing("round budget spent", &shell, &repo)
20328 .await
20329 .expect("stop_reviewing retry");
20330 assert_eq!(
20331 runner.state.reviews.len(),
20332 1,
20333 "no new round was started: {:?}",
20334 runner.state.reviews
20335 );
20336 let last = runner.state.reviews.last().expect("round record");
20337 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20338 assert!(
20339 last.verified_at.expect("still known") > first_attempt_at,
20340 "a second reentry must be a fresh attempt, not a stale copy of the first"
20341 );
20342 assert_ne!(runner.state.status, RunStatus::Blocked);
20343
20344 held.release();
20345 }
20346
20347 #[cfg(unix)]
20348 #[tokio::test]
20349 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20350 ask_test_home();
20351 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20352 let tmp = tempfile::tempdir().unwrap();
20353 let repo = tmp.path().join("repo");
20354 std::fs::create_dir(&repo).unwrap();
20355 init_repo(&repo);
20356 let run_git = |args: &[&str]| {
20357 let out = std::process::Command::new("git")
20358 .args(args)
20359 .current_dir(&repo)
20360 .output()
20361 .unwrap();
20362 assert!(
20363 out.status.success(),
20364 "{args:?}: {}",
20365 String::from_utf8_lossy(&out.stderr)
20366 );
20367 };
20368 let origin = tmp.path().join("origin.git");
20369 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20370 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20371 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20372 run_git(&["add", "spec.rs"]);
20373 run_git(&["commit", "-m", "initial spec"]);
20374 run_git(&["push", "origin", "main"]);
20375 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20376 run_git(&["checkout", "-b", "fix/review"]);
20377 std::fs::write(
20378 repo.join("check.rs"),
20379 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20380 )
20381 .unwrap();
20382 run_git(&["add", "check.rs"]);
20383 run_git(&["commit", "-m", "use spec"]);
20384 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20385 run_git(&["checkout", "main"]);
20386 std::fs::write(
20387 repo.join("spec.rs"),
20388 "struct AgentSpec { max_turn_secs: u64 }\n",
20389 )
20390 .unwrap();
20391 run_git(&["add", "spec.rs"]);
20392 run_git(&["commit", "-m", "extend spec"]);
20393 run_git(&["push", "origin", "main"]);
20394 run_git(&["checkout", "fix/review"]);
20395
20396 let mut runner = runner_at(RunStatus::Reviewing);
20397 runner.state.repo = repo.clone();
20398 runner.state.base_commit = base;
20399 runner.state.pr_head = Some("fix/review".into());
20400 runner.state.config.graph.review_rounds = max_rounds;
20401 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20402 runner.state.config.graph.sessions = false;
20403 runner.state.config.disk.min_free_bytes = 0;
20404 runner.state.config.verify.e2e =
20405 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20406 runner.state.candidates = vec![Candidate {
20407 index: 0,
20408 label: 'A',
20409 agent: "fixture".into(),
20410 branch: "fix/review".into(),
20411 worktree: repo.clone(),
20412 summary: String::new(),
20413 stat: String::new(),
20414 files: 1,
20415 commits: 1,
20416 empty: false,
20417 failed: None,
20418 verified_noop: None,
20419 duration_ms: 0,
20420 folded: false,
20421 }];
20422 runner.state.tally = Some(Tally {
20423 first_choice: BTreeMap::new(),
20424 borda: BTreeMap::new(),
20425 winner: 'A',
20426 rankings: 0,
20427 unanimous_initial: true,
20428 deliberated: false,
20429 changed_votes: 0,
20430 unanimous_final: true,
20431 tie_break: None,
20432 judges: 0,
20433 present: 0,
20434 quorum: 0,
20435 met_quorum: true,
20436 uncontested: Some("review-only".into()),
20437 });
20438 let mut last = review_round(false, 0, 1, 1, false, false);
20439 last.head = reviewed.clone();
20440 last.e2e_deferred = true;
20441 last.e2e[0].resource_blocked = true;
20442 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20443 if max_rounds == 1 {
20444 last.e2e.clear();
20445 }
20446 runner.state.reviews = vec![last];
20447 let calls = tmp.path().join("fix-calls");
20448 let mut fixer = spec("repair");
20449 fixer
20450 .env
20451 .insert("FIX_CALLS".into(), calls.display().to_string());
20452 fixer.command = vec!["sh".into(), "-c".into(), r#"
20453 echo fix >> "$FIX_CALLS"
20454 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20455 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20456 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20457 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20458 "#.into()];
20459 let mut reviewer = spec("reviewer");
20460 reviewer.command = vec![
20461 "sh".into(),
20462 "-c".into(),
20463 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20464 .into(),
20465 ];
20466 runner.roles.fixer = Some(vec![fixer]);
20467 runner.roles.reviewers = vec![reviewer];
20468
20469 runner.sync_to_base().await.unwrap();
20470 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20471 assert_ne!(rebased, reviewed);
20472 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20473 if reentry {
20474 let shell = runner.state.config.shell();
20477 runner
20478 .stop_unresolved_review("deferred verification", &shell, &repo)
20479 .await
20480 .unwrap();
20481 assert!(runner.catchup_repairable());
20482 assert!(!calls.exists());
20483 runner.state = RunState::load(&runner.state.id).unwrap();
20484 assert!(runner.catchup_repairable());
20485 }
20486 runner.review_loop().await.unwrap();
20487 let first = &runner.state.reviews[0];
20488 assert_eq!(first.head, reviewed);
20489 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20490 assert!(
20491 first.verify_retried,
20492 "a reproduced build failure gets exactly one retry: {:?}",
20493 first.e2e
20494 );
20495 assert!(first.e2e[0].output_tail.contains("E0063"));
20496 assert!(!runner.catchup_repairable());
20497 if max_rounds == 2 {
20498 assert_eq!(
20499 runner.state.status,
20500 RunStatus::Gating,
20501 "{}",
20502 crate::report::run(&runner.state)
20503 );
20504 assert_eq!(runner.state.reviews.len(), 2);
20505 assert!(first.fix.as_ref().unwrap().committed);
20506 assert!(runner.state.reviews[1].clean);
20507 assert_eq!(
20508 runner
20509 .state
20510 .reviews
20511 .iter()
20512 .filter(|r| r.fix.is_some())
20513 .count(),
20514 1
20515 );
20516 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20517 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20518 } else {
20519 assert_eq!(runner.state.status, RunStatus::Blocked);
20520 assert!(first.fix.is_none());
20521 assert!(!calls.exists());
20522 let recovery = crate::recovery::assess(
20523 "prior",
20524 Some(&runner.state),
20525 &runner.landing_base(),
20526 crate::recovery::Relation::Diverged,
20527 );
20528 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20529 let reasons = recovery.reasons.join("\n");
20530 assert!(
20531 reasons.contains(&reviewed) && reasons.contains(&rebased),
20532 "{reasons}"
20533 );
20534 assert!(
20535 reasons.contains("base sync to")
20536 && reasons.contains("additional review is required"),
20537 "{reasons}"
20538 );
20539 }
20540 }
20541 }
20542
20543 #[tokio::test]
20555 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20556 crate::run::pin_test_home();
20557 let home = crate::run::home();
20558
20559 let tmp = tempfile::tempdir().expect("tempdir");
20560 let repo = tmp.path().join("repo");
20561 std::fs::create_dir_all(&repo).unwrap();
20562 init_repo(&repo);
20563 let head = crate::git::rev_parse(&repo, "HEAD")
20564 .await
20565 .expect("rev-parse");
20566 let cache_dir = tmp.path().join("target");
20567
20568 let mut config = Config::default();
20569 config.verify.e2e = vec![format!(
20570 "CARGO_TARGET_DIR='{}' test -f README.md",
20571 cache_dir.display()
20572 )];
20573 config.graph.review_rounds = 1;
20574 config.graph.timeout_verify = Some(2);
20575 config.graph.timeout_resource_wait = 2;
20576
20577 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20578 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20579 .expect("no io error acquiring directly")
20580 {
20581 crate::cache::AcquireOutcome::Acquired(g) => g,
20582 crate::cache::AcquireOutcome::Busy(b) => {
20583 panic!("expected the direct acquire to win the lease first: {b:?}")
20584 }
20585 };
20586
20587 let mut state = RunState::new(
20588 repo.clone(),
20589 "main".to_owned(),
20590 head.clone(),
20591 "task".to_owned(),
20592 config,
20593 );
20594 state.candidates = vec![Candidate {
20595 index: 0,
20596 label: 'A',
20597 agent: "alpha".to_owned(),
20598 branch: "does-not-exist".to_owned(),
20599 worktree: repo.clone(),
20600 summary: String::new(),
20601 stat: String::new(),
20602 files: 0,
20603 commits: 0,
20604 empty: false,
20605 failed: None,
20606 verified_noop: None,
20607 duration_ms: 0,
20608 folded: false,
20609 }];
20610 state.tally = Some(Tally {
20611 first_choice: BTreeMap::from([('A', 1)]),
20612 borda: BTreeMap::new(),
20613 winner: 'A',
20614 rankings: 1,
20615 unanimous_initial: true,
20616 deliberated: false,
20617 changed_votes: 0,
20618 unanimous_final: true,
20619 tie_break: None,
20620 judges: 0,
20621 present: 0,
20622 quorum: 0,
20623 met_quorum: true,
20624 uncontested: Some("only candidate A produced a change".to_owned()),
20625 });
20626 state.reviews = vec![ReviewRound {
20630 round: 1,
20631 head: head.clone(),
20632 verified_head: Some(head.clone()),
20633 verified_at: Some(jiff::Timestamp::now()),
20634 reviews: Vec::new(),
20635 e2e: vec![CommandOutcome {
20636 command: format!(
20637 "CARGO_TARGET_DIR='{}' test -f README.md",
20638 cache_dir.display()
20639 ),
20640 code: None,
20641 output_tail: "waiting for the shared build cache".to_owned(),
20642 duration_ms: 0,
20643 resource_blocked: true,
20644 }],
20645 fix: None,
20646 blocking: 1,
20647 answered: 1,
20648 expected: 1,
20649 clean: false,
20650 verify_retried: false,
20651 e2e_deferred: false,
20652 e2e_defer_reason: None,
20653 progressed: false,
20654 vote_split: false,
20655 reconsideration: Vec::new(),
20656 verdict: None,
20657 }];
20658
20659 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20660 let mut runner = Runner {
20661 state,
20662 roles: ResolvedRoles {
20663 implementers: Vec::new(),
20664 judges: Vec::new(),
20665 reviewers: Vec::new(),
20666 fixer: None,
20667 conductor: conductor(),
20668 implementer_roster: Vec::new(),
20669 judge_roster: Vec::new(),
20670 reviewer_roster: Vec::new(),
20671 },
20672 sem: Arc::new(Semaphore::new(1)),
20673 pause: Pause::new(),
20674 interrupt: Pause::new(),
20675 };
20676
20677 runner.review_loop().await.expect("review_loop");
20682
20683 assert_eq!(
20684 runner.state.reviews.len(),
20685 1,
20686 "no new round was started on top of the unresolved one: {:?}",
20687 runner.state.reviews
20688 );
20689 let last = &runner.state.reviews[0];
20690 assert_eq!(
20691 last.e2e_status(),
20692 E2eStatus::ResourceBlocked,
20693 "still contended: {last:?}"
20694 );
20695 assert!(
20696 last.verified_at.expect("still known") > first_attempt_at,
20697 "review_loop must have actually retried the check, not left it exactly as found"
20698 );
20699 assert_ne!(
20700 runner.state.status,
20701 RunStatus::Blocked,
20702 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20703 runner.state.status
20704 );
20705
20706 held.release();
20707 }
20708
20709 #[tokio::test]
20710 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20711 crate::run::pin_test_home();
20712 for status in [RunStatus::Blocked, RunStatus::Gating] {
20713 let mut runner = runner_at(status);
20714 runner.state.config.graph.land = true;
20715 runner.state.gate_ran = true;
20716 runner.state.merge = Some(MergeOutcome {
20717 mode: MergeMode::Pr,
20718 ok: false,
20719 detail: "the fixer produced no commit".into(),
20720 empty: false,
20721 });
20722 let url = "https://example.invalid/o/r/pull/1";
20723 runner.state.pr = Some(crate::run::PrRecord {
20724 url: url.into(),
20725 number: 1,
20726 state: "open".into(),
20727 checks: "red".into(),
20728 round: 1,
20729 rounds: 1,
20730 red_at_merge: Vec::new(),
20731 });
20732 runner.state.save().unwrap();
20733 runner.execute_graph().await.unwrap();
20737 assert_eq!(runner.state.status, RunStatus::Blocked);
20738 assert!(runner.state.land_sync_pending);
20739 assert!(runner.state.events.iter().any(|event|
20740 event.node == "land" && event.message == format!("watching {url}")));
20741 assert_eq!(
20742 runner.state.merge.as_ref().unwrap().detail,
20743 "the fixer produced no commit"
20744 );
20745 }
20746 }
20747
20748 #[tokio::test]
20749 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20750 crate::run::pin_test_home();
20751 let tmp = tempfile::tempdir().expect("tempdir");
20752 let repo = tmp.path().join("repo");
20753 std::fs::create_dir_all(&repo).unwrap();
20754 init_repo(&repo);
20755
20756 let mut config = Config::default();
20757 config.merge.mode = MergeMode::Pr;
20758 config.graph.land = true;
20759 config.graph.land_approval = false;
20760
20761 let mut state = RunState::new(
20762 repo.clone(),
20763 "main".to_owned(),
20764 "deadbeef".to_owned(),
20765 "task".to_owned(),
20766 config,
20767 );
20768 state.candidates = vec![Candidate {
20769 index: 0,
20770 label: 'A',
20771 agent: "alpha".to_owned(),
20772 branch: "does-not-exist".to_owned(),
20773 worktree: repo.clone(),
20774 summary: String::new(),
20775 stat: String::new(),
20776 files: 0,
20777 commits: 0,
20778 empty: false,
20779 failed: None,
20780 verified_noop: None,
20781 duration_ms: 0,
20782 folded: false,
20783 }];
20784 state.tally = Some(Tally {
20785 first_choice: BTreeMap::from([('A', 1)]),
20786 borda: BTreeMap::new(),
20787 winner: 'A',
20788 rankings: 1,
20789 unanimous_initial: true,
20790 deliberated: false,
20791 changed_votes: 0,
20792 unanimous_final: true,
20793 tie_break: None,
20794 judges: 0,
20795 present: 0,
20796 quorum: 0,
20797 met_quorum: true,
20798 uncontested: Some("only candidate A produced a change".to_owned()),
20799 });
20800 state.reviews = vec![ReviewRound {
20801 round: 1,
20802 head: "deadbeef".to_owned(),
20803 verified_head: None,
20804 verified_at: None,
20805 reviews: Vec::new(),
20806 e2e: Vec::new(),
20807 fix: None,
20808 blocking: 0,
20809 answered: 0,
20810 expected: 0,
20811 clean: true,
20812 verify_retried: false,
20813 e2e_deferred: false,
20814 e2e_defer_reason: None,
20815 progressed: false,
20816 vote_split: false,
20817 reconsideration: Vec::new(),
20818 verdict: None,
20819 }];
20820 state.gate = vec![CommandOutcome {
20821 command: "test".to_owned(),
20822 code: Some(0),
20823 output_tail: String::new(),
20824 duration_ms: 0,
20825 resource_blocked: false,
20826 }];
20827 state.gate_ran = true;
20828 state.status = RunStatus::Landing;
20832 state.merge = Some(MergeOutcome {
20833 mode: MergeMode::Pr,
20834 ok: true,
20835 detail: "https://example.invalid/x/y/pull/1".to_owned(),
20836 empty: false,
20837 });
20838
20839 ask_test_home();
20843 state.save().expect("save landing run");
20846 let store = ask::Questions::open();
20847 let q = ask_open_question(&store, &state.id);
20848
20849 let mut runner = Runner {
20850 state,
20851 roles: ResolvedRoles {
20852 implementers: Vec::new(),
20853 judges: Vec::new(),
20854 reviewers: Vec::new(),
20855 fixer: None,
20856 conductor: conductor(),
20857 implementer_roster: Vec::new(),
20858 judge_roster: Vec::new(),
20859 reviewer_roster: Vec::new(),
20860 },
20861 sem: Arc::new(Semaphore::new(1)),
20862 pause: Pause::new(),
20863 interrupt: Pause::new(),
20864 };
20865
20866 runner.execute().await.expect("execute");
20871
20872 assert_eq!(
20873 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20874 Some("https://example.invalid/x/y/pull/1"),
20875 "reentry must not push again or open a second pull request over the \
20876 one `land` is already watching"
20877 );
20878 assert_ne!(
20879 runner.state.status,
20880 RunStatus::Landing,
20881 "land could not actually reach the fake pull request, so it must \
20882 have given up rather than left the run silently parked forever"
20883 );
20884 assert_eq!(runner.state.status, RunStatus::Blocked);
20888 assert!(
20889 store.get(&q.id).unwrap().status.open(),
20890 "Blocked is still alive; settle_questions must have been a no-op here"
20891 );
20892 }
20893
20894 fn state_with_round(round: ReviewRound) -> RunState {
20895 let mut s = RunState::new(
20896 PathBuf::from("/repo"),
20897 "main".to_owned(),
20898 "abc1234".to_owned(),
20899 "add retries".to_owned(),
20900 Config::default(),
20901 );
20902 s.reviews = vec![round];
20903 s
20904 }
20905
20906 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
20907 crate::verdict::Finding {
20908 id: id.to_owned(),
20909 severity,
20910 file: None,
20911 line: None,
20912 title: title.to_owned(),
20913 detail: String::new(),
20914 }
20915 }
20916
20917 #[test]
20918 fn pr_body_names_open_findings_and_declined_ones() {
20919 let round = ReviewRound {
20920 round: 2,
20921 head: "deadbee".to_owned(),
20922 verified_head: None,
20923 verified_at: None,
20924 reviews: vec![ReviewRecord {
20925 attempts: 0,
20926 reviewer: 1,
20927 agent: "alpha".to_owned(),
20928 summary: String::new(),
20929 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
20930 vote: None,
20931 failed: None,
20932 duration_ms: 0,
20933 }],
20934 e2e: vec![CommandOutcome {
20935 command: "cargo test".to_owned(),
20936 code: Some(0),
20937 output_tail: String::new(),
20938 duration_ms: 0,
20939 resource_blocked: false,
20940 }],
20941 verify_retried: false,
20942 e2e_deferred: false,
20943 e2e_defer_reason: None,
20944 fix: Some(FixRecord {
20945 agent: "alpha".to_owned(),
20946 addressed: Vec::new(),
20947 rejected: vec![crate::verdict::Rejection {
20948 id: "R1-1-1".to_owned(),
20949 why: "not reachable from any caller".to_owned(),
20950 }],
20951 notes: String::new(),
20952 committed: true,
20953 failed: None,
20954 duration_ms: 0,
20955 continuation: None,
20956 }),
20957 blocking: 0,
20958 answered: 1,
20959 expected: 1,
20960 clean: false,
20961 progressed: true,
20962 vote_split: false,
20963 reconsideration: Vec::new(),
20964 verdict: None,
20965 };
20966 let state = state_with_round(round);
20967 let body = pr_message(&state, 'A').body;
20968
20969 assert!(body.contains("add retries"), "the task must still be there");
20970 assert!(body.contains("R2-1-1"), "{body}");
20971 assert!(!body.contains("unused import"), "{body}");
20972 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
20973 assert!(
20974 !body.contains("not reachable from any caller"),
20975 "the reason it was declined: {body}"
20976 );
20977 }
20978
20979 #[test]
20980 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
20981 let round = ReviewRound {
20982 round: 1,
20983 head: "deadbee".to_owned(),
20984 verified_head: None,
20985 verified_at: None,
20986 reviews: vec![ReviewRecord {
20987 attempts: 0,
20988 reviewer: 1,
20989 agent: "alpha".to_owned(),
20990 summary: String::new(),
20991 findings: Vec::new(),
20992 vote: None,
20993 failed: None,
20994 duration_ms: 0,
20995 }],
20996 e2e: Vec::new(),
20997 verify_retried: false,
20998 e2e_deferred: false,
20999 e2e_defer_reason: None,
21000 fix: None,
21001 blocking: 0,
21002 answered: 1,
21003 expected: 1,
21004 clean: true,
21005 progressed: false,
21006 vote_split: false,
21007 reconsideration: Vec::new(),
21008 verdict: None,
21009 };
21010 let state = state_with_round(round);
21011 let body = pr_message(&state, 'A').body;
21012 assert!(!body.contains("Open review findings"), "{body}");
21013 assert!(!body.contains("Declined"), "{body}");
21014 }
21015
21016 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21017 let mut state = RunState::new(
21018 PathBuf::from("/repo"),
21019 "main".to_owned(),
21020 "abc1234".to_owned(),
21021 instruction.to_owned(),
21022 Config::default(),
21023 );
21024 state.candidates.push(Candidate {
21025 index: 0,
21026 label: 'A',
21027 agent: "alpha".to_owned(),
21028 branch: "magi/x/A".to_owned(),
21029 worktree: PathBuf::from("/wt"),
21030 summary: summary.to_owned(),
21031 stat: String::new(),
21032 files: 1,
21033 commits: 1,
21034 empty: false,
21035 failed: None,
21036 verified_noop: None,
21037 folded: false,
21038 duration_ms: 0,
21039 });
21040 state
21041 }
21042
21043 fn review_state(subjects: &[&str]) -> RunState {
21044 let mut state = state_with_summary(
21045 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21046 "",
21047 );
21048 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21049 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21050 state
21051 }
21052
21053 #[tokio::test]
21056 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21057 ask_test_home();
21058 let tmp = tempfile::tempdir().unwrap();
21059 let repo = tmp.path().join("repo");
21060 std::fs::create_dir_all(&repo).unwrap();
21061 init_repo(&repo);
21062 let origin = tmp.path().join("origin.git");
21063 let g = |dir: &Path, args: &[&str]| {
21064 let out = std::process::Command::new("git")
21065 .args(args)
21066 .current_dir(dir)
21067 .quiet()
21068 .output()
21069 .expect("spawn git");
21070 assert!(
21071 out.status.success(),
21072 "git {args:?}: {}",
21073 String::from_utf8_lossy(&out.stderr)
21074 );
21075 };
21076 g(
21077 tmp.path(),
21078 &[
21079 "clone",
21080 "--bare",
21081 "-q",
21082 repo.to_str().unwrap(),
21083 origin.to_str().unwrap(),
21084 ],
21085 );
21086 g(
21087 &repo,
21088 &["remote", "add", "origin", origin.to_str().unwrap()],
21089 );
21090 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21091
21092 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21094 g(&repo, &["add", "-A"]);
21095 g(
21096 &repo,
21097 &["commit", "-q", "-m", "chore(deps): update a crate"],
21098 );
21099 g(&repo, &["push", "-q", "origin", "main"]);
21100 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21101 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21102 g(&repo, &["add", "-A"]);
21103 g(
21104 &repo,
21105 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21106 );
21107 g(&repo, &["checkout", "-q", "main"]);
21108
21109 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21110 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21111 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21113 assert_eq!(fallback, c1);
21114
21115 let mut state = review_state(&[
21116 "chore(deps): update a crate",
21117 "fix(daemon): apply a chosen action",
21118 ]);
21119 state.repo = repo.clone();
21120 state.base_branch = "main".to_owned();
21121 state.base_commit = c1;
21122 refresh_reviewed_commits(&mut state, "feat/own").await;
21123 assert_eq!(
21124 state.reviewed_commits,
21125 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21126 );
21127 assert_eq!(
21128 review_title(&state).as_deref(),
21129 Some("fix(daemon): apply a chosen action")
21130 );
21131 assert_eq!(
21132 leaked_subjects(&state, "feat/own").await,
21133 Some(vec!["chore(deps): update a crate".to_owned()])
21134 );
21135 state.config.merge.remote = "nowhere".to_owned();
21138 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21139 }
21140
21141 struct SummaryFixture {
21145 _tmp: tempfile::TempDir,
21146 repo: PathBuf,
21147 reply: PathBuf,
21148 runner: Runner,
21149 }
21150
21151 fn summary_fixture() -> SummaryFixture {
21152 ask_test_home();
21153 let tmp = tempfile::tempdir().unwrap();
21154 let repo = tmp.path().join("repo");
21155 std::fs::create_dir_all(&repo).unwrap();
21156 init_repo(&repo);
21157 let origin = tmp.path().join("origin.git");
21158 let g = |args: &[&str]| {
21159 let out = std::process::Command::new("git")
21160 .args(args)
21161 .current_dir(&repo)
21162 .quiet()
21163 .output()
21164 .expect("spawn git");
21165 assert!(
21166 out.status.success(),
21167 "git {args:?}: {}",
21168 String::from_utf8_lossy(&out.stderr)
21169 );
21170 };
21171 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21172 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21173 g(&["push", "-q", "origin", "main"]);
21174 g(&["checkout", "-q", "-b", "feat/own"]);
21175 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21176 g(&["add", "-A"]);
21177 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21178 let reply = tmp.path().join("reply.txt");
21179 std::fs::write(
21180 &reply,
21181 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21182 )
21183 .unwrap();
21184 let mut runner = runner_at(RunStatus::Gating);
21185 let mut state = review_state(&["feat(own): add the own file"]);
21186 state.repo = repo.clone();
21187 state.base_branch = "main".to_owned();
21188 state.candidates[0].branch = "feat/own".to_owned();
21189 state.candidates[0].worktree = repo.clone();
21190 let mut author = spec("alpha");
21191 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21192 author
21193 .env
21194 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21195 state.config.agents = vec![author];
21196 runner.state = state;
21197 SummaryFixture {
21198 _tmp: tmp,
21199 repo,
21200 reply,
21201 runner,
21202 }
21203 }
21204
21205 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21206 let winner = f.runner.state.candidates[0].clone();
21207 f.runner
21208 .guarded_pr_message(&winner, None, true)
21209 .await
21210 .unwrap()
21211 .expect("an English description never parks")
21212 }
21213
21214 fn clean_round_with_minor_findings() -> ReviewRound {
21215 let mut round = passing_round(2, None);
21216 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21217 round.reviews = vec![ReviewRecord {
21218 attempts: 0,
21219 reviewer: 1,
21220 agent: "alpha".into(),
21221 summary: "内部レビュー".into(),
21222 findings: vec![
21223 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21224 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21225 ],
21226 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21227 failed: None,
21228 duration_ms: 0,
21229 }];
21230 round
21231 }
21232
21233 #[tokio::test]
21234 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21235 for review_only in [false, true] {
21236 let mut f = summary_fixture();
21237 if !review_only {
21238 f.runner.state.candidates[0].agent = "alpha".into();
21239 }
21240 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21241 assert!(f.runner.state.reviews[0].clean);
21242 assert!(f.runner.state.reviews[0].fix.is_none());
21243 assert!(f.runner.state.open_findings().is_empty());
21244 assert_eq!(
21245 crate::pr_outcome::required_finding_ids(&f.runner.state),
21246 ["R2-1-1", "R2-1-2"]
21247 );
21248 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21249 let evidence = crate::pr_outcome::summary_evidence(
21250 &f.runner.state,
21251 crate::pr_outcome::FinalHead::At(&head),
21252 );
21253 assert!(
21254 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21255 );
21256 let omitted = posted(&mut f).await;
21258 assert!(omitted.body.contains("No description"));
21259 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21260 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21261 assert!(f.runner.state.change_summary.is_none());
21262 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21263 let described = posted(&mut f).await;
21264 assert!(described.body.contains("improve documentation for callers"));
21265 assert!(described.body.contains("remaining edge case"));
21266 assert!(described.body.contains("## Open review findings"));
21267 assert!(!described.body.contains("未修正"));
21268 }
21269 }
21270
21271 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21272 let mut f = summary_fixture();
21273 if !review_only {
21274 f.runner.state.candidates[0].agent = "alpha".into();
21275 }
21276 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21277 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21278 fixed
21279 .fix
21280 .as_mut()
21281 .unwrap()
21282 .rejected
21283 .push(crate::verdict::Rejection {
21284 id: "R1-1-1".into(),
21285 why: "呼び出されないコードのため辞退".into(),
21286 });
21287 let mut clean = clean_round_with_minor_findings();
21288 clean.verified_head = Some(head.clone());
21289 f.runner.state.reviews = vec![fixed, clean];
21290 ok_gate(&mut f.runner.state, Some(&head));
21291 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21292 f.runner.state.config.agents.push(scripted("beta", reply));
21293 f.runner.state.config.roles.language_judge =
21294 Some(crate::config::AgentChoice::One("beta".into()));
21295 f
21296 }
21297
21298 #[tokio::test]
21299 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21300 for review_only in [false, true] {
21301 let mut f = rewrite_fixture_with_findings(review_only,
21302 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21303 let m = posted(&mut f).await;
21304 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21305 let expected =
21306 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21307 assert!(m.body.contains(&expected));
21308 assert!(m.body.contains("## Review rounds"));
21309 assert!(
21310 m.body
21311 .contains("Gate (`verify.gate`): passed on the final commit")
21312 );
21313 assert!(
21314 m.body
21315 .contains("Tests (`verify.e2e`): passed on the final commit")
21316 );
21317 let c = f.runner.state.change_summary.as_ref().unwrap();
21318 assert!(
21319 m.body
21320 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21321 );
21322 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21323 assert!(crate::pr_outcome::covers_required_findings(
21324 &f.runner.state,
21325 &m.body
21326 ));
21327 assert!(m.body.contains("documentation remains incomplete"));
21328 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21329 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21330 assert!(f.runner.state.github_text.is_none());
21331 }
21332 }
21333
21334 #[tokio::test]
21335 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21336 for review_only in [false, true] {
21337 for reply in [
21338 "TITLE: fix: public description\nA short English description.",
21339 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21340 ] {
21341 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21342 let winner = f.runner.state.candidates[0].clone();
21343 let posted = f
21344 .runner
21345 .guarded_pr_message(&winner, None, true)
21346 .await
21347 .unwrap();
21348 assert!(posted.is_none());
21349 assert!(f.runner.state.parked);
21350 let withheld = f.runner.state.github_text.as_ref().unwrap();
21351 assert!(
21352 withheld.rewrite.is_none(),
21353 "an incomplete rewrite must not be selectable"
21354 );
21355 let q = ask::Questions::open()
21356 .get(withheld.question.as_ref().unwrap())
21357 .unwrap();
21358 assert!(
21359 !q.choices
21360 .contains(&crate::github_text::USE_REWRITE.to_owned())
21361 );
21362 assert!(
21363 q.detail.contains("R1-1-1")
21364 && q.detail.contains("R2-1-1")
21365 && q.detail.contains("R2-1-2")
21366 );
21367 }
21368 }
21369 }
21370
21371 #[tokio::test]
21372 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21373 for review_only in [false, true] {
21374 let mut f = summary_fixture();
21375 if !review_only {
21376 f.runner.state.candidates[0].agent = "alpha".into();
21377 }
21378 f.runner.state.instruction = "内部タスクの説明".into();
21379 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21380 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21381 f.runner.state.reviews = vec![
21382 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21383 round_with_notes(2, Some("二回目の内部修正報告です。")),
21384 passing_round(3, Some(&head)),
21385 ];
21386 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21387 ok_gate(&mut f.runner.state, Some(&head));
21388 let captured = f.repo.join("public-prompt.txt");
21389 let author = &mut f.runner.state.config.agents[0];
21390 author.command = vec![
21391 "sh".into(),
21392 "-c".into(),
21393 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21394 ];
21395 author.env.insert(
21396 "PROMPT_FILE".into(),
21397 captured.to_string_lossy().into_owned(),
21398 );
21399 let m = posted(&mut f).await;
21400 assert!(m.body.contains("Adds `own.txt`"));
21401 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21402 assert!(m.body.contains("R1-1-1"));
21403 assert!(
21404 m.body
21405 .contains("Gate (`verify.gate`): passed on the final commit")
21406 );
21407 assert!(
21408 m.body
21409 .contains("Tests (`verify.e2e`): passed on the final commit")
21410 );
21411 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21412 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21413 let prompt = std::fs::read_to_string(captured).unwrap();
21414 assert!(prompt.contains(&format!(
21415 "git diff {} {head}",
21416 f.runner.state.change_summary.as_ref().unwrap().base
21417 )));
21418 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21419 assert_eq!(
21420 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21421 "二回目の内部修正報告です。"
21422 );
21423 }
21424 }
21425
21426 #[tokio::test]
21427 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21428 for review_only in [false, true] {
21429 let mut f = summary_fixture();
21430 if !review_only {
21431 f.runner.state.candidates[0].agent = "alpha".into();
21432 }
21433 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21434 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21435 std::fs::write(&f.reply, "no usable description").unwrap();
21436 let failed = posted(&mut f).await;
21437 assert!(
21438 failed
21439 .body
21440 .contains("No description of what this change does could be generated")
21441 );
21442 assert!(failed.body.is_ascii());
21443 assert!(f.runner.state.github_text.is_none());
21444 assert!(f.runner.state.change_summary.is_none());
21445 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21446 let evidence = crate::pr_outcome::summary_evidence(
21447 &f.runner.state,
21448 crate::pr_outcome::FinalHead::At(&head),
21449 );
21450 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21451 assert!(
21452 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21453 .exists()
21454 );
21455 f.runner.state =
21456 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21457 std::fs::write(
21458 &f.reply,
21459 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21460 )
21461 .unwrap();
21462 let retried = posted(&mut f).await;
21463 assert!(retried.body.contains("Adds the committed own file."));
21464 assert_eq!(
21465 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21466 "日本語の修正報告"
21467 );
21468 }
21469 }
21470
21471 #[tokio::test]
21472 async fn same_head_verification_and_findings_invalidate_public_prose() {
21473 let mut f = summary_fixture();
21474 posted(&mut f).await;
21475 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21476 let original = f
21477 .runner
21478 .state
21479 .change_summary
21480 .as_ref()
21481 .unwrap()
21482 .evidence
21483 .clone();
21484 ok_gate(&mut f.runner.state, Some(&head));
21485 std::fs::write(
21486 &f.reply,
21487 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21488 )
21489 .unwrap();
21490 let verified = posted(&mut f).await;
21491 assert!(verified.body.contains("now verified"));
21492 assert_ne!(
21493 f.runner.state.change_summary.as_ref().unwrap().evidence,
21494 original
21495 );
21496 let mut round = round_with_notes(1, Some("内部報告"));
21497 round.reviews.push(ReviewRecord {
21498 attempts: 0,
21499 reviewer: 1,
21500 agent: "alpha".into(),
21501 summary: "内部レビュー".into(),
21502 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21503 vote: None,
21504 failed: None,
21505 duration_ms: 0,
21506 });
21507 round
21508 .fix
21509 .as_mut()
21510 .unwrap()
21511 .rejected
21512 .push(crate::verdict::Rejection {
21513 id: "R1-1-1".into(),
21514 why: "呼び出し元から到達できない".into(),
21515 });
21516 f.runner.state.reviews = vec![round];
21517 let omitted = posted(&mut f).await;
21519 assert!(omitted.body.contains("No description"));
21520 assert!(omitted.body.contains("R1-1-1"));
21521 assert!(!omitted.body.contains("到達"));
21522 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21523 let described = posted(&mut f).await;
21524 assert!(described.body.contains("unreachable from callers"));
21525 assert!(described.body.contains("R1-1-1"));
21526 assert!(!described.body.contains("到達"));
21527 let evidence = f
21528 .runner
21529 .state
21530 .change_summary
21531 .as_ref()
21532 .unwrap()
21533 .evidence
21534 .clone();
21535 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21536 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21537 assert!(posted(&mut f).await.body.contains("rationale updated"));
21538 assert_ne!(
21539 f.runner.state.change_summary.as_ref().unwrap().evidence,
21540 evidence
21541 );
21542 f.runner
21544 .state
21545 .change_summary
21546 .as_mut()
21547 .unwrap()
21548 .evidence
21549 .clear();
21550 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21551 assert!(posted(&mut f).await.body.contains("current specification"));
21552 }
21553
21554 #[tokio::test]
21555 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21556 let mut f = summary_fixture();
21557 let m = posted(&mut f).await;
21558 assert_eq!(m.title, "feat(own): add the own file");
21559 assert!(
21560 m.body
21561 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21562 "{}",
21563 m.body
21564 );
21565 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21566 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21567 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21568 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21569 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21570 let kept = f.runner.state.change_summary.as_ref().unwrap();
21571 assert_eq!(
21572 (kept.head.as_str(), kept.base.as_str()),
21573 (head.as_str(), base.as_str())
21574 );
21575 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21576 }
21577
21578 #[tokio::test]
21579 async fn review_only_summary_receives_the_configured_github_language() {
21580 let mut f = summary_fixture();
21581 f.runner.state.config.graph.github_language = "ja".into();
21582 let captured = f.repo.join("summary-prompt.txt");
21583 let author = &mut f.runner.state.config.agents[0];
21584 author.command = vec![
21585 "sh".into(),
21586 "-c".into(),
21587 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21588 ];
21589 author.env.insert(
21590 "PROMPT_FILE".into(),
21591 captured.to_string_lossy().into_owned(),
21592 );
21593 std::fs::write(
21594 &f.reply,
21595 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21596 )
21597 .unwrap();
21598 let m = posted(&mut f).await;
21599 let prompt = std::fs::read_to_string(captured).unwrap();
21600 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21601 assert!(!prompt.contains("in English"));
21602 assert_eq!(m.title, "feat(own): ファイルを追加");
21603 assert!(m.body.contains("own.txt を追加します。"));
21604 }
21605
21606 #[tokio::test]
21607 async fn review_fixes_come_after_the_whole_change_summary() {
21608 let mut f = summary_fixture();
21609 f.runner.state.reviews = vec![
21610 round_with_notes(1, Some("handled the PATH case")),
21611 round_with_notes(2, None),
21612 ];
21613 let m = posted(&mut f).await;
21614 let summary = m.body.find("## Summary").expect("summary");
21615 let fixes = m.body.find("## Review rounds").expect("review rounds");
21616 assert!(summary < fixes, "{}", m.body);
21617 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21618 assert!(m.body.contains("## Final verification"), "{}", m.body);
21619 }
21620
21621 #[tokio::test]
21622 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21623 let mut f = summary_fixture();
21624 let first = posted(&mut f).await;
21625 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21626 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21628 let again = posted(&mut f).await;
21629 assert_eq!((again.title, again.body), (first.title, first.body));
21630
21631 let g = |args: &[&str]| {
21632 let out = std::process::Command::new("git")
21633 .args(args)
21634 .current_dir(&f.repo)
21635 .quiet()
21636 .output()
21637 .expect("spawn git");
21638 assert!(out.status.success(), "git {args:?}");
21639 };
21640 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21641 g(&["add", "-A"]);
21642 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21643 let second = posted(&mut f).await;
21644 let kept = f.runner.state.change_summary.clone().unwrap();
21645 assert_ne!(kept.head, first_head);
21646 assert!(second.body.contains("Not read yet."), "{}", second.body);
21647 assert!(!second.body.contains("so the first summary is read"));
21648 assert_eq!(second.title, "feat(own): other");
21649 }
21650
21651 #[tokio::test]
21652 async fn an_unavailable_summary_is_stated_and_never_invented() {
21653 let mut f = summary_fixture();
21654 std::fs::write(&f.reply, "no title line here").unwrap();
21655 let m = posted(&mut f).await;
21656 assert!(
21657 m.body
21658 .contains("No description of what this change does could be generated"),
21659 "{}",
21660 m.body
21661 );
21662 assert!(m.body.contains("internal reports remain in local records"));
21663 assert!(f.runner.state.change_summary.is_none());
21664 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21665 let winner = f.runner.state.candidates[0].clone();
21667 let preview = f
21668 .runner
21669 .guarded_pr_message(&winner, None, false)
21670 .await
21671 .unwrap()
21672 .unwrap();
21673 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21674 }
21675
21676 #[test]
21677 fn pr_message_review_single_commit_uses_its_subject() {
21678 let state = review_state(&["feat(nats): per-role user"]);
21679 let m = pr_message(&state, 'A');
21680 assert_eq!(m.title, "feat(nats): per-role user");
21681 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21682 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21683 }
21684
21685 #[test]
21686 fn pr_message_review_multi_commit_takes_the_oldest() {
21687 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21688 let m = pr_message(&state, 'A');
21689 assert_eq!(m.title, "feat: the change");
21690 for s in ["feat: the change", "fix: typo", "fix: again"] {
21691 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21692 }
21693 }
21694
21695 #[test]
21696 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21697 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21698 let mut state = review_state(&[first, "fix: later fixup"]);
21699 state.candidates[0].branch = "機能/ログイン".to_owned();
21700 let m = pr_message(&state, 'A');
21701 assert!(
21702 m.title.starts_with("chore: land candidate A of run"),
21703 "{}",
21704 m.title
21705 );
21706 }
21707 }
21708
21709 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21710 BranchFacts {
21711 commits: commits
21712 .iter()
21713 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21714 .collect(),
21715 stat: stat.to_owned(),
21716 }
21717 }
21718
21719 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21720 let mut r = review_round(true, 0, 1, 1, true, true);
21721 r.round = round;
21722 r.fix = notes.map(|n| FixRecord {
21723 agent: "fixer".to_owned(),
21724 addressed: Vec::new(),
21725 rejected: Vec::new(),
21726 notes: n.to_owned(),
21727 committed: true,
21728 failed: None,
21729 duration_ms: 0,
21730 continuation: None,
21731 });
21732 r
21733 }
21734
21735 #[test]
21736 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21737 let state = review_state(&["ignored"]);
21738 let f = facts(
21739 &[
21740 (
21741 "fix(login): resolve PATH on macOS",
21742 "Login shells skip rc files.",
21743 ),
21744 ("fix: address review", ""),
21745 ],
21746 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21747 );
21748 let m = pr_message_with(&state, 'A', Some(&f));
21749 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21750 assert!(
21751 m.body
21752 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
21753 "{}",
21754 m.body
21755 );
21756 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21757 assert!(m.body.contains("## Diff stat"), "{}", m.body);
21758 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21759 for banned in [
21760 "Review the work already",
21761 "no task statement",
21762 "Original task",
21763 ] {
21764 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21765 }
21766 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
21767 }
21768
21769 #[test]
21770 fn pr_message_review_truncates_a_huge_first_commit_body() {
21771 let state = review_state(&["ignored"]);
21772 let f = facts(
21773 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
21774 "s",
21775 );
21776 let m = pr_message_with(&state, 'A', Some(&f));
21777 assert!(m.body.len() < 30_000, "{}", m.body.len());
21778 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
21779 assert!(
21780 m.body.contains("- ... 1 more commit(s)"),
21781 "{}",
21782 m.body.len()
21783 );
21784 assert!(m.body.ends_with("magi:candidate-a\n"));
21785 }
21786
21787 #[test]
21788 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
21789 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
21790 assert_eq!(m.title, "feat: x");
21791 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
21792 assert!(!m.body.contains("Diff stat"), "{}", m.body);
21793 assert!(!m.body.contains("no task statement"), "{}", m.body);
21794 }
21795
21796 #[test]
21797 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
21798 let mut state = review_state(&["日本語の件名"]);
21799 state.candidates[0].branch = "fix/macos-login-path".to_owned();
21800 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
21801 state.candidates[0].branch = "機能/ログイン".to_owned();
21802 assert!(
21803 pr_message(&state, 'A')
21804 .title
21805 .starts_with("chore: land candidate A")
21806 );
21807 }
21808
21809 #[test]
21810 fn pr_message_review_fixes_survive_a_clean_final_round() {
21811 let mut state = review_state(&["feat: x"]);
21812 state.reviews = vec![
21813 round_with_notes(1, Some("handled the PATH case")),
21814 round_with_notes(2, None),
21815 ];
21816 let body = pr_message(&state, 'A').body;
21817 assert!(
21818 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
21819 "{body}"
21820 );
21821 assert!(body.contains("### Round 1"), "{body}");
21822
21823 state.reviews = vec![
21824 round_with_notes(1, Some("first fix")),
21825 round_with_notes(2, Some("")),
21826 round_with_notes(3, Some("second fix")),
21827 round_with_notes(4, None),
21828 ];
21829 let body = pr_message(&state, 'A').body;
21830 assert!(
21831 body.contains("### Round 1") && !body.contains("first fix"),
21832 "{body}"
21833 );
21834 assert!(
21835 body.contains("### Round 3") && !body.contains("second fix"),
21836 "{body}"
21837 );
21838 assert!(body.contains("### Round 2"), "{body}");
21839 }
21840
21841 const FINAL: &str = "abcdef1234567890";
21842
21843 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
21844 let mut r = round_with_notes(round, None);
21845 r.verified_head = verified.map(str::to_owned);
21846 r
21847 }
21848
21849 fn ok_gate(state: &mut RunState, head: Option<&str>) {
21850 state.gate = vec![CommandOutcome {
21851 command: "gate".to_owned(),
21852 code: Some(0),
21853 output_tail: String::new(),
21854 duration_ms: 0,
21855 resource_blocked: false,
21856 }];
21857 state.gate_ran = true;
21858 state.gate_head = head.map(str::to_owned);
21859 }
21860
21861 fn body_at(state: &RunState) -> String {
21862 pr_message_raw(
21863 state,
21864 'A',
21865 None,
21866 crate::pr_outcome::FinalHead::At(FINAL),
21867 ChangeNote::NotAsked,
21868 )
21869 .body
21870 }
21871
21872 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
21873
21874 #[test]
21875 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
21876 let mut state = review_state(&["feat: x"]);
21877 state.reviews = vec![
21878 round_with_notes(1, Some(SANDBOXED)),
21879 passing_round(2, Some(FINAL)),
21880 ];
21881 ok_gate(&mut state, Some(FINAL));
21882 let body = body_at(&state);
21883 assert!(!body.contains("Handled the PATH case."), "{body}");
21884 assert!(!body.contains("could not commit"), "{body}");
21885 assert!(!body.contains("could not run"), "{body}");
21886 assert!(
21887 body.contains("The fix is committed on the branch"),
21888 "{body}"
21889 );
21890 assert!(!body.contains("sandbox"), "{body}");
21891 assert!(
21892 body.contains("Tests (`verify.e2e`): passed on the final commit"),
21893 "{body}"
21894 );
21895 assert!(
21896 body.contains("Gate (`verify.gate`): passed on the final commit"),
21897 "{body}"
21898 );
21899 assert!(
21901 state.reviews[0]
21902 .fix
21903 .as_ref()
21904 .unwrap()
21905 .notes
21906 .contains("could not commit")
21907 );
21908 }
21909
21910 #[test]
21911 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
21912 let mut state = review_state(&["feat: x"]);
21913 let mut r1 = round_with_notes(1, Some(SANDBOXED));
21914 r1.fix.as_mut().unwrap().committed = false;
21915 let mut r2 = passing_round(2, Some(FINAL));
21916 r2.e2e[0].code = Some(1);
21917 state.reviews = vec![r1, r2];
21918 let body = body_at(&state);
21919 assert!(!body.contains("could not commit"), "{body}");
21920 assert!(!body.contains("could not run"), "{body}");
21921 assert!(body.contains("This round produced no commit"), "{body}");
21922 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
21923 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
21924 assert!(!body.contains("passed"), "{body}");
21925 }
21926
21927 #[test]
21928 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
21929 let mut state = review_state(&["feat: x"]);
21930 state.reviews = vec![
21931 round_with_notes(1, Some(SANDBOXED)),
21932 passing_round(2, Some("0123456789abcdef")),
21933 ];
21934 ok_gate(&mut state, Some("0123456789abcdef"));
21935 let body = body_at(&state);
21936 assert!(!body.contains("HTTP tests could not run"), "{body}");
21937 assert!(!body.contains("could not commit"), "{body}");
21939 assert!(
21940 body.contains("passed on an earlier commit (`0123456`)"),
21941 "{body}"
21942 );
21943 assert!(!body.contains("passed on the final commit"), "{body}");
21944
21945 ok_gate(&mut state, None);
21947 let body = body_at(&state);
21948 assert!(
21949 body.contains("passed on a commit that was not recorded"),
21950 "{body}"
21951 );
21952 }
21953
21954 #[test]
21955 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
21956 let state = state_with_summary(
21957 "add retries to the client",
21958 "TITLE: feat: retries\n\nDid it.",
21959 );
21960 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
21961 assert_eq!(m.title, "feat: retries");
21962 assert!(
21963 m.body.contains("<summary>Original task</summary>"),
21964 "{}",
21965 m.body
21966 );
21967 assert!(!m.body.contains("Commits under review"), "{}", m.body);
21968 assert!(
21969 m.body
21970 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
21971 "{}",
21972 m.body
21973 );
21974 }
21975
21976 #[test]
21977 fn pr_message_review_bounds_a_long_english_subject() {
21978 let long = format!("feat: {}", "word ".repeat(100));
21979 let m = pr_message(&review_state(&[&long]), 'A');
21980 assert!(m.title.starts_with("feat: word"), "{}", m.title);
21981 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21982 }
21983
21984 #[test]
21985 fn pr_message_implementation_run_is_unchanged_by_review_support() {
21986 let state = state_with_summary("add retries\n\ndetails", "- did some things");
21987 let m = pr_message(&state, 'A');
21988 assert_eq!(m.title, "add retries");
21989 assert!(m.body.contains("<summary>Original task</summary>"));
21990 assert!(!m.body.contains("Commits under review"));
21991 assert_eq!(landing_subject_source(&state), state.instruction);
21992 }
21993
21994 #[test]
21995 fn review_run_squash_subject_is_the_change_not_the_prompt() {
21996 let state = review_state(&["feat: the change", "fix: typo"]);
21997 let source = landing_subject_source(&state);
21998 assert_eq!(land::merge_subject("", &source), "feat: the change");
21999 assert_eq!(
22000 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22001 "feat: the change"
22002 );
22003 assert_eq!(
22005 land::merge_subject("feat: renamed by hand", &source),
22006 "feat: renamed by hand"
22007 );
22008 let blank = review_state(&["日本語"]);
22009 assert!(
22010 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22011 );
22012 }
22013
22014 #[test]
22015 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22016 let state = review_state(&["feat: the change"]);
22017 let source = landing_subject_source(&state);
22018 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22019 assert_eq!(
22020 land::merge_subject(landing_title(&state, old), &source),
22021 "feat: the change"
22022 );
22023 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22024 let task = state_with_summary("add retries", "");
22025 assert_eq!(landing_title(&task, old), old);
22026 }
22027
22028 #[test]
22029 fn pr_message_describes_the_change_not_the_task() {
22030 let state = state_with_summary(
22031 "今回やってほしいこと: results projector を直す",
22032 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22033 );
22034 let m = pr_message(&state, 'A');
22035 assert_eq!(m.title, "fix(web): batch the runs list reads");
22036 assert!(
22037 m.body.starts_with("## Summary\n\n- reads run.json once"),
22038 "{}",
22039 m.body
22040 );
22041 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22042 let task_at = m.body.find("今回やってほしいこと").unwrap();
22043 let details_at = m.body.find("<details>").unwrap();
22044 assert!(
22045 details_at < task_at,
22046 "the task lives inside <details>: {}",
22047 m.body
22048 );
22049 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22050 assert!(m.body.contains("magi:candidate-a"));
22051 }
22052
22053 #[test]
22054 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22055 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22056 let m = pr_message(&state, 'A');
22057 assert_eq!(m.title, "add retries");
22058 assert!(
22059 m.body.contains("## Summary\n\n- did some things"),
22060 "{}",
22061 m.body
22062 );
22063
22064 let none = RunState::new(
22065 PathBuf::from("/repo"),
22066 "main".to_owned(),
22067 "abc1234".to_owned(),
22068 "add retries".to_owned(),
22069 Config::default(),
22070 );
22071 let m = pr_message(&none, 'A');
22072 assert_eq!(m.title, "add retries");
22073 assert!(!m.body.contains("## Summary"), "{}", m.body);
22074 }
22075
22076 #[test]
22077 fn pr_message_refuses_the_candidate_commit_subject() {
22078 for bad in [
22079 "TITLE: magi: candidate A (uncommitted work)",
22080 "TITLE: chore: stuff (uncommitted work)",
22081 "TITLE: ",
22082 ] {
22083 let state = state_with_summary("add retries", bad);
22084 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22085 }
22086 }
22087
22088 #[test]
22089 fn pr_message_bounds_a_very_long_task_and_title() {
22090 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22091 let state = state_with_summary(&long, "- nothing");
22092 let m = pr_message(&state, 'A');
22093 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22094 assert!(!m.title.contains('\n'));
22095
22096 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22097 let m = pr_message(&state, 'A');
22098 assert!(m.title.starts_with("feat: "));
22099 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22100 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22101 }
22102
22103 fn long_title_of(instruction: &str) -> String {
22104 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22105 }
22106
22107 #[test]
22108 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22109 let first = "Make the landing path keep a readable title for long tasks";
22110 let line = format!(
22111 "{first}. {}",
22112 "And then keep going with more words ".repeat(20)
22113 );
22114 let t = long_title_of(&line);
22115 assert_eq!(t, first);
22116 assert!(!t.starts_with("chore: land"));
22117 }
22118
22119 #[test]
22120 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22121 let line = "word ".repeat(200);
22122 let t = long_title_of(&line);
22123 assert!(t.ends_with("word..."), "{t}");
22124 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22125 }
22126
22127 #[test]
22128 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22129 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22130 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22131 }
22132 }
22133
22134 #[test]
22135 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22136 let mut state = state_with_summary("再試行を追加する", "- nothing");
22137 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22138 state.config.graph.github_language = "ja".into();
22139 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22140 }
22141
22142 #[test]
22143 fn pr_message_title_limit_is_exact() {
22144 let at = "a".repeat(PR_TITLE_MAX);
22145 assert_eq!(long_title_of(&at), at);
22146 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22147 assert!(over.ends_with("..."), "{over}");
22148 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22149 }
22150
22151 #[test]
22152 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22153 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22154 let t = long_title_of(&line);
22155 assert!(t.ends_with("..."), "{t}");
22156 assert!(t.is_ascii(), "{t}");
22157 }
22158
22159 #[test]
22160 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22161 let line = format!(
22162 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22163 "plus more filler words ".repeat(20)
22164 );
22165 let t = long_title_of(&line);
22166 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22167 }
22168
22169 #[test]
22170 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22171 let line = format!("\n\n# {}", "title words ".repeat(40));
22172 let state = state_with_summary(&line, "- nothing");
22173 let m = pr_message(&state, 'A');
22174 assert!(m.title.starts_with("title words"), "{}", m.title);
22175 assert_eq!(
22176 land::merge_subject(&m.title, &landing_subject_source(&state)),
22177 m.title
22178 );
22179 assert_eq!(
22181 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22182 "feat: renamed by hand"
22183 );
22184 }
22185
22186 #[tokio::test]
22187 async fn github_text_rewrite_is_bounded_and_falls_back() {
22188 crate::run::pin_test_home();
22189 for (reply, accepted) in [
22190 (
22191 "TITLE: fix: retries\nAdd retries for failed requests.",
22192 true,
22193 ),
22194 (
22195 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22196 false,
22197 ),
22198 (
22199 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22200 true,
22201 ),
22202 (
22203 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22204 true,
22205 ),
22206 (
22207 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22208 true,
22209 ),
22210 (
22211 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22212 false,
22213 ),
22214 ("TITLE: fix: retries\n", false),
22215 ] {
22216 let dir = tempfile::tempdir().unwrap();
22217 let mut runner = runner_at(RunStatus::Gating);
22218 runner.state = state_with_summary(
22219 "add retries",
22220 "TITLE: fix: retries\n日本語の説明を書きます。",
22221 );
22222 runner.state.repo = dir.path().to_owned();
22223 runner.state.candidates[0].worktree = dir.path().to_owned();
22224 let mut author = spec("alpha");
22225 author.command = vec![
22226 "sh".into(),
22227 "-c".into(),
22228 "printf '%s' \"$REWRITE_REPLY\"".into(),
22229 ];
22230 author.env.insert("REWRITE_REPLY".into(), reply.into());
22231 runner.state.config.agents = vec![author];
22232 seed_public_summary(&mut runner);
22233 let winner = runner.state.candidates[0].clone();
22234 let first = runner
22235 .guarded_pr_message(&winner, None, true)
22236 .await
22237 .unwrap();
22238 let message = if accepted {
22239 first.expect("a passing rewrite never asks")
22240 } else {
22241 assert!(first.is_none());
22243 assert!(runner.state.parked);
22244 let store = ask::Questions::open();
22245 let asked: Vec<_> = store
22246 .list()
22247 .into_iter()
22248 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22249 .collect();
22250 assert_eq!(asked.len(), 1);
22251 assert!(!asked[0].detail.contains("secret"));
22252 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22253 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22254 assert_eq!(
22255 asked[0]
22256 .choices
22257 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22258 rewrite.is_some()
22259 );
22260 store
22263 .update(&asked[0].id, |q| {
22264 q.abandon("test");
22265 Ok(())
22266 })
22267 .unwrap();
22268 let again = runner
22269 .guarded_pr_message(&winner, None, true)
22270 .await
22271 .unwrap()
22272 .expect("an abandoned question falls back");
22273 let n = store
22274 .list()
22275 .into_iter()
22276 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22277 .count();
22278 assert_eq!(n, 1, "asked once per rejected text");
22279 again
22280 };
22281 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22282 assert!(!message.body.contains("secret"));
22283 assert!(!message.body.contains("prefix"));
22284 assert!(!message.title.contains("secret"));
22285 assert_eq!(
22286 message.body.contains("Add retries for failed requests."),
22287 accepted
22288 );
22289 assert_eq!(
22290 runner.state.seats["github-text-rewrite"].turns, 1,
22291 "only one rewrite invocation"
22292 );
22293 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22294 if accepted {
22295 assert!(runner.state.github_text.is_none());
22296 assert!(!runner.state.parked);
22297 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22298 assert!(message.body.contains("## Final verification"));
22299 assert!(message.body.contains("Evidence: final diff"));
22300 assert!(
22301 message
22302 .body
22303 .contains(&format!("magi:run/{}", runner.state.id))
22304 );
22305 }
22306 if !accepted {
22307 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22308 assert!(
22309 message
22310 .body
22311 .contains(&format!("magi:run/{}", runner.state.id))
22312 );
22313 }
22314 }
22315 }
22316
22317 fn scripted(id: &str, reply: &str) -> AgentSpec {
22318 let mut a = spec(id);
22319 a.command = vec![
22320 "sh".into(),
22321 "-c".into(),
22322 "printf '%s' \"$REWRITE_REPLY\"".into(),
22323 ];
22324 a.env.insert("REWRITE_REPLY".into(), reply.into());
22325 a
22326 }
22327
22328 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22329 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22330
22331 fn seed_public_summary(runner: &mut Runner) {
22333 let repo = &runner.state.repo;
22334 let g = |args: &[&str]| {
22335 let out = std::process::Command::new("git")
22336 .args(args)
22337 .current_dir(repo)
22338 .quiet()
22339 .output()
22340 .unwrap();
22341 assert!(out.status.success(), "git {args:?}");
22342 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22343 };
22344 g(&["init", "-q", "-b", "main"]);
22345 g(&[
22346 "-c",
22347 "user.name=Test",
22348 "-c",
22349 "user.email=test@example.invalid",
22350 "commit",
22351 "-q",
22352 "--allow-empty",
22353 "-m",
22354 "feat: fixture",
22355 ]);
22356 let head = g(&["rev-parse", "HEAD"]);
22357 g(&["update-ref", "refs/remotes/origin/main", &head]);
22358 let evidence = crate::pr_outcome::summary_evidence(
22359 &runner.state,
22360 crate::pr_outcome::FinalHead::At(&head),
22361 );
22362 let generated = &runner.state.candidates[0].summary;
22363 runner.state.change_summary = Some(crate::run::ChangeSummary {
22364 base: head.clone(),
22365 head,
22366 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22367 title: summary_title(generated).unwrap(),
22368 body: summary_without_title(generated),
22369 });
22370 }
22371
22372 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22373 let mut runner = runner_at(RunStatus::Gating);
22374 runner.state = state_with_summary("add retries", JA_SUMMARY);
22375 if review_only {
22376 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22377 }
22378 runner.state.repo = dir.to_owned();
22379 runner.state.candidates[0].worktree = dir.to_owned();
22380 runner.state.config.agents = agents;
22381 seed_public_summary(&mut runner);
22382 runner
22383 }
22384
22385 fn gate_questions(runner: &Runner) -> usize {
22386 ask::Questions::open()
22387 .list()
22388 .into_iter()
22389 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22390 .count()
22391 }
22392
22393 #[tokio::test]
22394 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22395 crate::run::pin_test_home();
22396 let dir = tempfile::tempdir().unwrap();
22397 let mut runner = rewrite_runner(
22398 dir.path(),
22399 true,
22400 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22401 );
22402 runner.state.config.roles.language_judge =
22403 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22404 let winner = runner.state.candidates[0].clone();
22405 let msg = runner
22406 .guarded_pr_message(&winner, None, true)
22407 .await
22408 .unwrap()
22409 .expect("an English rewrite posts without asking");
22410 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22411 assert!(msg.body.contains("<details>kept</details>"));
22412 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22413 assert_eq!(gate_questions(&runner), 0);
22414 assert!(runner.state.github_text.is_none());
22415 assert!(!runner.state.seats.contains_key("impl-A"));
22416 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22417 }
22418
22419 #[tokio::test]
22420 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22421 crate::run::pin_test_home();
22422 let dir = tempfile::tempdir().unwrap();
22423 let mut runner = rewrite_runner(
22424 dir.path(),
22425 false,
22426 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22427 );
22428 runner.state.config.roles.language_judge =
22429 Some(crate::config::AgentChoice::One("beta".into()));
22430 runner.state.seats.insert(
22431 "impl-A".into(),
22432 SeatState::new("impl-A", "alpha", runner.state.seed),
22433 );
22434 let winner = runner.state.candidates[0].clone();
22435 let msg = runner
22436 .guarded_pr_message(&winner, None, true)
22437 .await
22438 .unwrap()
22439 .expect("rewritten");
22440 assert!(msg.body.contains("<details>kept</details>"));
22441 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22442 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22443 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22444 }
22445
22446 #[tokio::test]
22447 async fn rewrite_reuses_a_resumable_implementer_session() {
22448 crate::run::pin_test_home();
22449 let dir = tempfile::tempdir().unwrap();
22450 let mut runner = rewrite_runner(
22451 dir.path(),
22452 false,
22453 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22454 );
22455 runner.state.config.roles.language_judge =
22456 Some(crate::config::AgentChoice::One("beta".into()));
22457 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22458 seat.turns = 1;
22459 runner.state.seats.insert("impl-A".into(), seat);
22460 let winner = runner.state.candidates[0].clone();
22461 runner
22462 .guarded_pr_message(&winner, None, true)
22463 .await
22464 .unwrap()
22465 .expect("rewritten");
22466 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22467 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22468 }
22469
22470 #[tokio::test]
22471 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22472 crate::run::pin_test_home();
22475 let dir = tempfile::tempdir().unwrap();
22476 let mut exhausted = spec("alpha");
22477 exhausted.command = vec!["false".into()];
22478 let mut runner = rewrite_runner(
22479 dir.path(),
22480 false,
22481 vec![exhausted, scripted("beta", EN_REWRITE)],
22482 );
22483 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22485 "alpha".into(),
22486 "beta".into(),
22487 ]));
22488 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22489 seat.turns = 1;
22490 runner.state.seats.insert("impl-A".into(), seat);
22491 let winner = runner.state.candidates[0].clone();
22492 let msg = runner
22493 .guarded_pr_message(&winner, None, true)
22494 .await
22495 .unwrap()
22496 .expect("the role rewrites after the implementer fails");
22497 assert!(msg.body.contains("`src/net.rs`"));
22498 assert_eq!(
22499 runner.state.seats["impl-A"].turns, 2,
22500 "the implementer was tried once"
22501 );
22502 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22503 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22504 }
22505
22506 #[tokio::test]
22507 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22508 crate::run::pin_test_home();
22509 let dir = tempfile::tempdir().unwrap();
22510 let mut broken = spec("bad");
22511 broken.command = vec!["false".into()];
22512 let mut runner = rewrite_runner(
22513 dir.path(),
22514 true,
22515 vec![
22516 broken,
22517 scripted("noformat", "Add retries."),
22518 scripted("good", EN_REWRITE),
22519 ],
22520 );
22521 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22522 "bad".into(),
22523 "noformat".into(),
22524 "good".into(),
22525 ]));
22526 let winner = runner.state.candidates[0].clone();
22527 let msg = runner
22528 .guarded_pr_message(&winner, None, true)
22529 .await
22530 .unwrap()
22531 .expect("the third agent rewrites");
22532 assert!(msg.body.contains("`src/net.rs`"));
22533 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22534 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22535 }
22536
22537 #[tokio::test]
22538 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22539 crate::run::pin_test_home();
22540 for (agents, role, saved) in [
22542 (
22543 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22544 Some("beta"),
22545 true,
22546 ),
22547 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22548 (Vec::new(), None, false),
22549 ] {
22550 let dir = tempfile::tempdir().unwrap();
22551 let mut runner = rewrite_runner(dir.path(), true, agents);
22552 runner.state.config.roles.language_judge =
22553 role.map(|r| crate::config::AgentChoice::One(r.into()));
22554 let winner = runner.state.candidates[0].clone();
22555 let first = runner
22556 .guarded_pr_message(&winner, None, true)
22557 .await
22558 .unwrap();
22559 assert!(first.is_none(), "withheld text parks the run");
22560 assert!(runner.state.parked);
22561 assert_eq!(gate_questions(&runner), 1);
22562 let g = runner.state.github_text.as_ref().unwrap();
22563 assert_eq!(g.rewrite.is_some(), saved);
22564 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22565 }
22566 }
22567
22568 #[tokio::test]
22569 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22570 crate::run::pin_test_home();
22571 let dir = tempfile::tempdir().unwrap();
22572 let mut runner = runner_at(RunStatus::Gating);
22573 runner.state =
22574 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22575 runner.state.repo = dir.path().to_owned();
22576 runner.state.candidates[0].worktree = dir.path().to_owned();
22577 let mut author = spec("alpha");
22578 author.command = vec![
22579 "sh".into(),
22580 "-c".into(),
22581 "printf '%s' \"$REWRITE_REPLY\"".into(),
22582 ];
22583 author.env.insert(
22584 "REWRITE_REPLY".into(),
22585 format!(
22586 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22587 "English sentence. ".repeat(600)
22588 ),
22589 );
22590 runner.state.config.agents = vec![author];
22591 seed_public_summary(&mut runner);
22592 let winner = runner.state.candidates[0].clone();
22593 assert!(
22594 runner
22595 .guarded_pr_message(&winner, None, true)
22596 .await
22597 .unwrap()
22598 .is_none()
22599 );
22600 let record = runner.state.github_text.clone().unwrap();
22601 let rewrite = record.rewrite.clone().unwrap();
22602 assert_eq!(rewrite.failed, ["body-language"]);
22603 assert!(
22604 rewrite
22605 .body
22606 .ends_with(&format!("magi:run/{}", runner.state.id))
22607 );
22608 let store = ask::Questions::open();
22609 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22610 assert!(
22611 q.choices
22612 .contains(&crate::github_text::USE_REWRITE.to_owned())
22613 );
22614 for label in [
22615 "ORIGINAL",
22616 "FALLBACK",
22617 "REWRITE",
22618 "Rules still failed: body-language",
22619 "truncated",
22620 ] {
22621 assert!(q.detail.contains(label), "{label}");
22622 }
22623 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22624 &runner.state,
22625 &crate::github_text::artifact_name(&record.fingerprint),
22626 ))
22627 .unwrap();
22628 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22629 runner.state =
22631 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22632 store
22633 .update(&q.id, |q| {
22634 q.say("まだ日本語")?;
22635 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22636 })
22637 .unwrap();
22638 assert!(
22639 runner
22640 .guarded_pr_message(&winner, None, true)
22641 .await
22642 .unwrap()
22643 .is_none()
22644 );
22645 let retry = runner.state.github_text.as_ref().unwrap();
22646 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22647 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22648 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22649 assert!(
22650 q.posting_comparison
22651 .as_ref()
22652 .unwrap()
22653 .matches_detail(&q.detail)
22654 );
22655 store
22656 .update(&q.id, |q| {
22657 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22658 })
22659 .unwrap();
22660 let message = runner
22661 .guarded_pr_message(&winner, None, true)
22662 .await
22663 .unwrap()
22664 .unwrap();
22665 assert_eq!(message.title, rewrite.title);
22666 assert_eq!(message.body, rewrite.body);
22667 assert_eq!(
22668 runner.state.github_text.as_ref().unwrap().choice,
22669 Some(GithubTextChoice::Rewrite)
22670 );
22671 let again = runner
22672 .guarded_pr_message(&winner, None, true)
22673 .await
22674 .unwrap()
22675 .unwrap();
22676 assert_eq!(again.title, message.title);
22677 assert_eq!(again.body, message.body);
22678 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22679 assert!(
22680 !runner
22681 .state
22682 .events
22683 .iter()
22684 .any(|e| e.message.contains(&rewrite.title)
22685 || e.message.contains("English sentence."))
22686 );
22687 }
22688
22689 #[tokio::test]
22690 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22691 crate::run::pin_test_home();
22692 let dir = tempfile::tempdir().unwrap();
22693 let mut runner = runner_at(RunStatus::Gating);
22694 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22695 runner.state.repo = dir.path().to_owned();
22696 runner.state.candidates[0].worktree = dir.path().to_owned();
22697 seed_public_summary(&mut runner);
22698 let winner = runner.state.candidates[0].clone();
22699 assert!(
22700 runner
22701 .guarded_pr_message(&winner, None, true)
22702 .await
22703 .unwrap()
22704 .is_none()
22705 );
22706 let record = runner.state.github_text.as_ref().unwrap();
22707 assert!(record.rewrite.is_none());
22708 let mut old = serde_json::to_value(record).unwrap();
22710 old.as_object_mut().unwrap().remove("rewrite");
22711 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22712 assert!(old.rewrite.is_none());
22713 let id = record.question.clone().unwrap();
22714 ask::Questions::open()
22715 .update(&id, |q| {
22716 q.choices.push(crate::github_text::USE_REWRITE.into());
22717 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22718 })
22719 .unwrap();
22720 let message = runner
22721 .guarded_pr_message(&winner, None, true)
22722 .await
22723 .unwrap()
22724 .unwrap();
22725 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22726 assert_eq!(
22727 runner.state.github_text.as_ref().unwrap().choice,
22728 Some(GithubTextChoice::Fallback)
22729 );
22730 }
22731
22732 #[tokio::test]
22733 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22734 crate::run::pin_test_home();
22735 let dir = tempfile::tempdir().unwrap();
22736 let mut runner = runner_at(RunStatus::Gating);
22737 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22738 runner.state.repo = dir.path().to_owned();
22739 runner.state.candidates[0].worktree = dir.path().to_owned();
22740 seed_public_summary(&mut runner);
22741 let winner = runner.state.candidates[0].clone();
22742 assert!(
22743 runner
22744 .guarded_pr_message(&winner, None, true)
22745 .await
22746 .unwrap()
22747 .is_none()
22748 );
22749 let store = ask::Questions::open();
22750 let find = |run: &str| {
22751 store
22752 .list()
22753 .into_iter()
22754 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22755 .unwrap()
22756 };
22757 let run = runner.state.id.clone();
22758 let q = find(&run);
22759 let fp = runner
22760 .state
22761 .github_text
22762 .as_ref()
22763 .unwrap()
22764 .fingerprint
22765 .clone();
22766 let name = crate::github_text::artifact_name(&fp);
22767 let path = crate::run::artifact_path(&runner.state, &name);
22768 assert!(q.detail.contains(&path.display().to_string()));
22769 assert!(q.detail.contains("title-language: title"));
22770 let saved = std::fs::read_to_string(&path).expect("artifact written");
22771 assert!(saved.contains("日本語のタイトル"));
22772 assert!(q.cwd.is_none());
22773 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
22774 let say = |id: &str, text: &str, choice: &str| {
22776 store
22777 .update(id, |q| {
22778 q.thread.push(ask::Turn {
22779 who: ask::Who::Operator,
22780 body: text.to_owned(),
22781 at: jiff::Timestamp::now(),
22782 note: None,
22783 });
22784 q.answer(ask::Answer::Choice(choice.to_owned()))
22785 })
22786 .unwrap();
22787 };
22788 say(&q.id, "まだ日本語", "use my text");
22789 assert!(
22790 runner
22791 .guarded_pr_message(&winner, None, true)
22792 .await
22793 .unwrap()
22794 .is_none()
22795 );
22796 let second = find(&run);
22797 assert_ne!(second.id, q.id);
22798 assert!(second.detail.contains("did not pass"));
22799 assert!(second.detail.contains("日本語のタイトル"));
22802 assert!(!second.detail.contains("まだ日本語"));
22803 assert!(second.detail.contains(&path.display().to_string()));
22804 say(&second.id, "fix: retry failed requests", "use my text");
22805 let message = runner
22806 .guarded_pr_message(&winner, None, true)
22807 .await
22808 .unwrap()
22809 .expect("a vetted replacement is posted");
22810 assert_eq!(message.title, "fix: retry failed requests");
22811 assert_eq!(
22812 runner
22813 .state
22814 .github_text
22815 .as_ref()
22816 .unwrap()
22817 .chosen_title
22818 .as_deref(),
22819 Some("fix: retry failed requests")
22820 );
22821 assert!(runner.state.github_text.as_ref().unwrap().resolved);
22822 let again = runner
22824 .guarded_pr_message(&winner, None, true)
22825 .await
22826 .unwrap()
22827 .unwrap();
22828 assert_eq!(again.title, message.title);
22829 assert_eq!(
22830 store
22831 .list()
22832 .into_iter()
22833 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
22834 .count(),
22835 2
22836 );
22837 }
22838
22839 #[test]
22840 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
22841 let mut state = state_with_summary(
22845 "add retries",
22846 "TITLE: fix(web): batch reads\n- reads run.json once",
22847 );
22848 state.config.graph.language = "ja".to_owned();
22849 let m = pr_message(&state, 'A');
22850 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
22851 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
22852
22853 let task = "今回やってほしいこと: results projector を直す";
22856 let mut state = state_with_summary(task, "- no title line");
22857 state.config.graph.language = "ja".to_owned();
22858 let m = pr_message(&state, 'A');
22859 assert_eq!(
22860 m.title,
22861 format!("chore: land candidate A of run {}", state.id)
22862 );
22863 assert!(
22864 m.body.contains(&format!(
22865 "<summary>Original task</summary>\n\n{task}\n\n</details>"
22866 )),
22867 "{}",
22868 m.body
22869 );
22870 }
22871
22872 #[test]
22873 fn pr_message_scrubs_home_paths_and_addresses() {
22874 let state = state_with_summary(
22875 "fix it in /Users/someone/src/x",
22876 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
22877 );
22878 let m = pr_message(&state, 'A');
22879 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
22880 assert!(!m.body.contains(leak), "{}", m.body);
22881 }
22882 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
22883 }
22884
22885 #[test]
22886 fn pr_message_survives_a_task_that_closes_details() {
22887 let state = state_with_summary("a </details> b", "TITLE: fix: x");
22888 let m = pr_message(&state, 'A');
22889 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
22890 }
22891
22892 #[test]
22893 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
22894 let cmd = manual_merge_command(
22895 MergeStyle::Squash,
22896 Path::new("/repo"),
22897 "b",
22898 "fix: \"quoted\" $(x) `y`\n\nbody",
22899 );
22900 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
22901 }
22902
22903 #[test]
22904 fn manual_merge_command_matches_the_configured_style() {
22905 let repo = Path::new("/repo");
22906 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
22907
22908 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
22909 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
22910
22911 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
22912 assert_eq!(
22913 squash,
22914 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
22915 \"Merge magi run 0832 (candidate A)\""
22916 );
22917
22918 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
22919 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
22920 }
22921
22922 #[test]
22923 fn a_nudge_gets_a_quarter_of_the_budget() {
22924 assert_eq!(retry_budget(secs(1200), true), secs(300));
22926 assert_eq!(retry_budget(secs(3600), true), secs(900));
22927 }
22928
22929 #[test]
22930 fn a_resent_prompt_keeps_the_whole_budget() {
22931 assert_eq!(retry_budget(secs(1200), false), secs(1200));
22934 assert_eq!(retry_budget(secs(60), false), secs(60));
22935 }
22936
22937 #[test]
22938 fn the_floor_never_exceeds_the_original_budget() {
22939 assert_eq!(retry_budget(secs(60), true), secs(60));
22943 assert_eq!(retry_budget(secs(480), true), secs(120));
22944 assert_eq!(retry_budget(secs(0), true), secs(0));
22945 }
22946
22947 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
22948 agent::CommandEvidence {
22949 id: "item1".to_owned(),
22950 description: "cargo test".to_owned(),
22951 exit_code,
22952 result_summary: String::new(),
22953 source: "codex".to_owned(),
22954 }
22955 }
22956
22957 #[test]
22958 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
22959 assert!(!has_unconfirmed_command(&[]));
22963 }
22964
22965 #[test]
22966 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
22967 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
22971 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
22972 assert!(!has_unconfirmed_command(&[
22973 evidence(Some(0)),
22974 evidence(Some(101))
22975 ]));
22976 }
22977
22978 #[test]
22979 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
22980 assert!(has_unconfirmed_command(&[
22981 evidence(Some(0)),
22982 evidence(None)
22983 ]));
22984 }
22985
22986 #[test]
22987 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
22988 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
22989 assert_eq!(
22990 verified_noop_claim(true, &[], text).as_deref(),
22991 Some("already fixed by b32cfc4, on main.")
22992 );
22993 }
22994
22995 #[test]
22996 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
22997 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23000 assert!(verified_noop_claim(false, &[], text).is_none());
23001 }
23002
23003 #[test]
23004 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23005 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23006 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23007 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23009 }
23010
23011 #[test]
23012 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23013 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23014 }
23015
23016 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23019 runner.state.candidates = shape
23020 .iter()
23021 .enumerate()
23022 .map(|(i, &(empty, verified))| Candidate {
23023 index: i,
23024 label: (b'A' + i as u8) as char,
23025 agent: "sonnet".to_owned(),
23026 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23027 worktree: PathBuf::from(format!("/wt/{i}")),
23028 summary: String::new(),
23029 stat: String::new(),
23030 files: 0,
23031 commits: 0,
23032 empty,
23033 failed: None,
23034 verified_noop: verified.map(str::to_owned),
23035 duration_ms: 0,
23036 folded: false,
23037 })
23038 .collect();
23039 }
23040
23041 #[test]
23042 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23043 ask_test_home();
23044 let mut runner = runner_at(RunStatus::Implementing);
23045 set_candidates(
23046 &mut runner,
23047 &[
23048 (true, Some("already on main at b32cfc4")),
23049 (true, Some("same fix, see the existing test")),
23050 ],
23051 );
23052
23053 runner
23054 .after_implement()
23055 .expect("a verified no-op is not an error");
23056
23057 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23058 }
23059
23060 #[test]
23061 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23062 ask_test_home();
23063 let mut runner = runner_at(RunStatus::Implementing);
23064 set_candidates(
23068 &mut runner,
23069 &[(true, Some("already on main at b32cfc4")), (true, None)],
23070 );
23071
23072 let err = runner
23073 .after_implement()
23074 .expect_err("an unverified empty candidate must still fail the run");
23075
23076 assert!(
23077 err.to_string().contains("no candidate produced a change"),
23078 "{err}"
23079 );
23080 assert_eq!(runner.state.status, RunStatus::Failed);
23081 }
23082
23083 #[test]
23084 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23085 ask_test_home();
23086 let mut runner = runner_at(RunStatus::Implementing);
23087 set_candidates(&mut runner, &[(true, None), (true, None)]);
23088
23089 let err = runner
23090 .after_implement()
23091 .expect_err("no candidate declared anything; this is an ordinary failure");
23092
23093 assert!(
23094 err.to_string().contains("no candidate produced a change"),
23095 "{err}"
23096 );
23097 assert_eq!(runner.state.status, RunStatus::Failed);
23098 }
23099}
23100
23101#[derive(Debug)]
23104struct GraftRestoreFailed(String);
23105
23106impl std::fmt::Display for GraftRestoreFailed {
23107 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23108 write!(
23109 f,
23110 "could not restore the winner to its pre-graft commit: {}",
23111 self.0
23112 )
23113 }
23114}
23115
23116impl std::error::Error for GraftRestoreFailed {}
23117
23118#[cfg(test)]
23122mod resource_wait_tests {
23123 use super::*;
23124 use std::collections::BTreeMap;
23125
23126 fn sh(id: &str, script: &str) -> AgentSpec {
23127 AgentSpec {
23128 id: id.to_owned(),
23129 kind: crate::config::AgentKind::Command,
23130 model: None,
23131 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23132 extra_args: Vec::new(),
23133 env: BTreeMap::new(),
23134 prompt_delivery: None,
23135 max_turn_secs: None,
23136 }
23137 }
23138
23139 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23140 SeatJob {
23141 seat: SeatState::new("impl-A", &spec.id, 1),
23142 spec,
23143 cwd: dir.to_path_buf(),
23144 prompt: "do it".to_owned(),
23145 timeout,
23146 allow_write: true,
23147 sessions: false,
23148 artifacts: dir.join("art"),
23149 stem: "impl-A".to_owned(),
23150 handover: None,
23151 }
23152 }
23153
23154 fn state(resource_wait_secs: u64) -> RunState {
23155 let mut config = Config::default();
23156 config.graph.timeout_resource_wait = resource_wait_secs;
23157 RunState::new(
23158 PathBuf::from("/repo"),
23159 "main".to_owned(),
23160 "abc1234".to_owned(),
23161 "task".to_owned(),
23162 config,
23163 )
23164 }
23165
23166 fn hold(cache: &Path) -> crate::cache::Guard {
23167 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23168 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23169 crate::cache::AcquireOutcome::Acquired(g) => g,
23170 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23171 }
23172 }
23173
23174 async fn run(
23175 jobs: Vec<SeatJob>,
23176 sem: Arc<Semaphore>,
23177 cache: Option<&Path>,
23178 st: &mut RunState,
23179 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23180 let prompts = Prompts::default();
23181 let ctx = WaveCtx {
23182 run: "r",
23183 node: "implement",
23184 prompts: &prompts,
23185 cache,
23186 round: None,
23187 carry_seats: false,
23188 pauses: None,
23189 };
23190 wave(jobs, sem, &ctx, st, 0).await
23191 }
23192
23193 #[tokio::test]
23194 async fn verification_cache_wait_preserves_the_command_budget() {
23195 crate::run::pin_test_home();
23196 let tmp = tempfile::tempdir().unwrap();
23197 let cache = tmp.path().join("target");
23198 let guard = hold(&cache);
23199 let release = tokio::spawn(async move {
23200 tokio::time::sleep(Duration::from_millis(100)).await;
23201 guard.release();
23202 });
23203 let mut st = state(10);
23204 let budget = Duration::from_millis(50);
23205 let wait = repository_wait(&st, None);
23206 let (outcomes, _) = with_cache_lease(
23207 &mut st,
23208 Some(&cache),
23209 "gate",
23210 "gate",
23211 tmp.path(),
23212 "abc1234",
23213 budget,
23214 "test gate",
23215 &wait,
23216 |_, available| async move {
23217 assert_eq!(
23218 available, budget,
23219 "contention must not shorten verification"
23220 );
23221 (Vec::new(), false, Vec::new())
23222 },
23223 )
23224 .await
23225 .unwrap();
23226 release.await.unwrap();
23227 assert!(outcomes.is_empty(), "{outcomes:?}");
23228 assert!(
23229 st.events
23230 .iter()
23231 .any(|event| event.message.contains("waiting for"))
23232 );
23233 assert!(st.resource_wait.is_none());
23234 hold(&cache).release();
23236 }
23237
23238 #[tokio::test]
23239 async fn verification_cache_wait_expires_without_starting_commands() {
23240 crate::run::pin_test_home();
23241 let tmp = tempfile::tempdir().unwrap();
23242 let cache = tmp.path().join("target");
23243 let _held = hold(&cache);
23244 let mut st = state(0);
23245 let wait = repository_wait(&st, None);
23246 let (outcomes, _) = with_cache_lease(
23247 &mut st,
23248 Some(&cache),
23249 "gate",
23250 "gate",
23251 tmp.path(),
23252 "abc1234",
23253 Duration::from_secs(60),
23254 "test gate",
23255 &wait,
23256 |_, _| async { panic!("commands must not start without the cache lease") },
23257 )
23258 .await
23259 .unwrap();
23260 assert_eq!(outcomes.len(), 1);
23261 assert!(outcomes[0].resource_blocked);
23262 assert!(st.resource_wait.is_none());
23263 assert!(matches!(
23264 st.resource_block.unwrap().cause,
23265 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23266 ));
23267 }
23268
23269 #[tokio::test]
23270 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23271 crate::run::pin_test_home();
23272 let tmp = tempfile::tempdir().unwrap();
23273 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23274 let cache = tmp.path().join("cache");
23275 let mut state = state(21600);
23276 let wait = repository_wait(&state, None);
23277 let (outcomes, _) = with_cache_lease(
23278 &mut state,
23279 Some(&cache),
23280 "gate",
23281 "gate",
23282 tmp.path(),
23283 "head",
23284 Duration::from_secs(60),
23285 "gate",
23286 &wait,
23287 |_, _| async { panic!("freshness failure must not start verification") },
23288 )
23289 .await
23290 .unwrap();
23291 assert_eq!(outcomes.len(), 1);
23292 assert!(outcomes[0].resource_blocked);
23293 let block = state.resource_block.unwrap();
23294 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23295 assert!(!block.description().contains("timed out"));
23296 }
23297
23298 #[tokio::test]
23299 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23300 crate::run::pin_test_home();
23301 let tmp = tempfile::tempdir().unwrap();
23302 let cache = tmp.path().join("cache");
23303 let _held = hold(&cache);
23304 let mut state = state(21600);
23305 let pause = Pause::new();
23306 let interrupt = Pause::new();
23307 let cause = ParkCause::Operator;
23308 pause.park_for(cause.clone());
23309 let prompts = Prompts::default();
23310 let ctx = WaveCtx {
23311 run: "test",
23312 node: "fix",
23313 prompts: &prompts,
23314 cache: Some(&cache),
23315 round: None,
23316 carry_seats: false,
23317 pauses: Some((&pause, &interrupt)),
23318 };
23319 let result = wave(
23320 vec![job(
23321 sh("writer", "exit 99"),
23322 tmp.path(),
23323 Duration::from_secs(1),
23324 )],
23325 Arc::new(Semaphore::new(1)),
23326 &ctx,
23327 &mut state,
23328 0,
23329 )
23330 .await;
23331 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23332 assert_eq!(result[0].1.turns, 0);
23333 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23334 assert_eq!(
23335 state.resource_block.unwrap().cause,
23336 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23337 );
23338 assert!(state.active.is_empty());
23339 }
23340
23341 #[tokio::test]
23342 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23343 crate::run::pin_test_home();
23344 let tmp = tempfile::tempdir().unwrap();
23345 let cache = tmp.path().join("target-a");
23346 let guard = hold(&cache);
23347 tokio::spawn(async move {
23349 tokio::time::sleep(Duration::from_millis(1500)).await;
23350 guard.release();
23351 });
23352 let mut st = state(60);
23353 let results = run(
23354 vec![job(
23355 sh("alpha", "echo ok"),
23356 tmp.path(),
23357 Duration::from_secs(1),
23358 )],
23359 Arc::new(Semaphore::new(1)),
23360 Some(&cache),
23361 &mut st,
23362 )
23363 .await;
23364 assert!(
23365 matches!(results[0].2, AgentOutcome::Ok(_)),
23366 "the wait must not be charged to the agent"
23367 );
23368 assert!(st.resource_block.is_none());
23369 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23370 assert!(st.resource_wait.is_none() && st.active.is_empty());
23371 }
23372
23373 #[tokio::test]
23374 #[cfg(unix)]
23375 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23376 use std::os::fd::AsRawFd as _;
23377 crate::run::pin_test_home();
23378 let tmp = tempfile::tempdir().unwrap();
23379 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23380 let common = tmp.path().join(".git");
23381 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23382 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23383 assert_eq!(
23384 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23385 0
23386 );
23387 let mut st = state(1);
23388 st.config.coordination = crate::config::test_coordination();
23389 let mut reader = job(
23390 sh("reader", "echo must-not-run"),
23391 tmp.path(),
23392 Duration::from_secs(30),
23393 );
23394 reader.allow_write = false;
23395 let results = run(
23396 vec![reader.clone()],
23397 Arc::new(Semaphore::new(1)),
23398 None,
23399 &mut st,
23400 )
23401 .await;
23402 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23403 assert_eq!(results[0].1.turns, 0);
23404 assert!(FailClass::of(&results[0].2).is_none());
23405 assert_eq!(
23406 st.resource_block.as_ref().unwrap().kind,
23407 "repository coordination"
23408 );
23409 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23410 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23411 drop(external);
23412 let editor = crate::repo_lock::SeatLease::for_checkout(
23413 tmp.path(),
23414 true,
23415 crate::config::test_coordination().as_ref(),
23416 )
23417 .await
23418 .unwrap();
23419 let pause = Pause::new();
23420 let interrupt = Pause::new();
23421 let prompts = Prompts::default();
23422 let ctx = WaveCtx {
23423 run: "r",
23424 node: "review",
23425 prompts: &prompts,
23426 cache: None,
23427 round: None,
23428 carry_seats: false,
23429 pauses: Some((&pause, &interrupt)),
23430 };
23431 let (results, ()) = tokio::join!(
23432 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23433 async {
23434 tokio::time::sleep(Duration::from_millis(50)).await;
23435 pause.park();
23436 }
23437 );
23438 assert!(
23439 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23440 );
23441 assert_eq!(results[0].1.turns, 0);
23442 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23443 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23444 drop(editor);
23445 }
23446
23447 #[tokio::test]
23448 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23449 crate::run::pin_test_home();
23450 let tmp = tempfile::tempdir().unwrap();
23451 let cache = tmp.path().join("target-b");
23452 let _held = hold(&cache);
23453 let mut st = state(1);
23454 let results = run(
23455 vec![job(
23456 sh("alpha", "echo ok"),
23457 tmp.path(),
23458 Duration::from_secs(1),
23459 )],
23460 Arc::new(Semaphore::new(1)),
23461 Some(&cache),
23462 &mut st,
23463 )
23464 .await;
23465 let (_, seat, out) = &results[0];
23466 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23467 assert!(wave_blocked(&results));
23468 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23469 assert_eq!(seat.turns, 0, "nothing was asked");
23470 assert!(
23471 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23472 "no agent was spawned"
23473 );
23474 assert!(FailClass::of(out).is_none(), "not a failure class");
23475 let block = st.resource_block.as_ref().expect("recorded for the resume");
23476 assert_eq!(block.kind, "shared build cache");
23477 assert!(matches!(
23478 block.cause,
23479 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23480 ));
23481 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23482 assert!(st.active.is_empty());
23483 }
23484
23485 #[tokio::test]
23486 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23487 crate::run::pin_test_home();
23488 let tmp = tempfile::tempdir().unwrap();
23489 let mut st = state(1);
23490 let results = run(
23491 vec![job(
23492 sh("alpha", "echo ok"),
23493 tmp.path(),
23494 Duration::from_secs(1),
23495 )],
23496 Arc::new(Semaphore::new(0)),
23497 None,
23498 &mut st,
23499 )
23500 .await;
23501 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23502 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23503 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23504 }
23505
23506 #[tokio::test]
23507 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23508 crate::run::pin_test_home();
23509 let tmp = tempfile::tempdir().unwrap();
23510 let mut st = state(60);
23511 let results = run(
23512 vec![job(
23513 sh("alpha", "sleep 30"),
23514 tmp.path(),
23515 Duration::from_secs(1),
23516 )],
23517 Arc::new(Semaphore::new(1)),
23518 None,
23519 &mut st,
23520 )
23521 .await;
23522 let out = &results[0].2;
23523 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23524 let class = FailClass::of(out).expect("a failure");
23525 assert_eq!(class, FailClass::Timeout);
23526 assert!(should_hand_over(None, &class), "existing handover policy");
23527 assert!(st.resource_block.is_none());
23528 }
23529}