1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 if state.startup.as_ref().is_some_and(|s| !s.complete)
1470 || (state.status == RunStatus::Prep
1471 && state.instruction.trim().is_empty()
1472 && state.candidates.is_empty()
1473 && state.prep_complete == Some(false))
1474 {
1475 bail!(
1476 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477 );
1478 }
1479 crate::pr_head::ensure_owner(&state)?;
1480 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481 bail!(
1482 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483 );
1484 }
1485 if let Some(to) = &state.released_to {
1486 bail!(
1487 "run {} cannot be resumed: its worktree was released to run {}",
1488 state.short(),
1489 crate::run::short_of(to)
1490 );
1491 }
1492 let roles = state.config.resolve_roles()?;
1493 if let Some(id) = &state.config.graph.selected_implementer
1494 && state.candidates.iter().any(|c| c.agent != *id)
1495 {
1496 bail!(
1497 "replacement implementer pin does not match the saved candidates; task must stay held"
1498 );
1499 }
1500 let max_parallel = state.config.graph.max_parallel.max(1);
1501 Ok(Self {
1502 state,
1503 roles,
1504 sem: Arc::new(Semaphore::new(max_parallel)),
1505 pause: Pause::new(),
1506 interrupt: Pause::new(),
1507 })
1508 }
1509
1510 pub async fn execute(&mut self) -> Result<()> {
1517 anyhow::ensure!(
1518 self.state.startup.as_ref().is_none_or(|s| s.complete),
1519 "startup incomplete; inspect retained evidence before explicit recovery"
1520 );
1521 if let Some(selection) = &self.state.pr_recovery
1522 && (selection.source_proof.is_some() || selection.source_run.is_some())
1523 {
1524 Box::pin(selection.verify_source(&self.state.repo)).await?;
1525 }
1526 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529 || self.state.land_approval.is_some()
1530 || self.state.merge_decision.is_some()
1531 || !self.state.merge_repairs.is_empty()
1532 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533 {
1534 let claim = FixClaim::acquire(&self.state.dir())?;
1535 let disk = RunState::load(&self.state.id)?;
1536 let checkpoint = |s: &RunState| {
1537 (
1538 s.merge_repairs.len(),
1539 s.merge_repairs
1540 .last()
1541 .map(|r| (r.question.clone(), r.phase)),
1542 s.land_approval
1543 .as_ref()
1544 .map(|a| (a.question.clone(), a.head.clone())),
1545 )
1546 };
1547 if checkpoint(&disk) != checkpoint(&self.state) {
1548 bail!(
1549 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550 );
1551 }
1552 Some(claim)
1553 } else {
1554 None
1555 };
1556 let mut execution = self.claim_driver().await?;
1557 let runner = &mut *execution.runner;
1558 let wait = runner.repository_wait();
1559 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562 let result = match result {
1563 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564 record_repository_block(&mut runner.state, &e);
1565 runner.park_here()?;
1566 Ok(())
1567 }
1568 other => other,
1569 };
1570 execution.driver.finish(&mut runner.state).await?;
1571 let ended = if result.is_err() {
1572 Some({
1573 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574 if let Err(error) = &result {
1575 notice.message = format!(
1576 "Run {} stopped with an error: {}",
1577 runner.state.id,
1578 error.to_string().lines().next().unwrap_or("unknown error")
1579 );
1580 }
1581 notice
1582 })
1583 } else {
1584 crate::notices::run_ended(&runner.state)
1585 };
1586 if let Some(notice) = ended {
1587 crate::notices::raise(notice);
1588 }
1589 result
1590 }
1591
1592 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594 }
1595
1596 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597 let wait = self.repository_wait();
1598 coordinate_operation(&mut self.state, cwd, &wait).await
1599 }
1600
1601 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603 let owner = match crate::pr_head::owner_lock_wait(
1604 &crate::run::home(),
1605 &self.state.id,
1606 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607 )
1608 .await
1609 {
1610 Ok(owner) => owner,
1611 Err(error) => {
1612 let detail = format!("worktree maintenance or ownership update: {error:#}");
1613 self.state.resource_block = Some(crate::run::ResourceBlock {
1617 cause: crate::run::ResourceBlockCause::Other,
1618 node: "start".into(),
1619 kind: "worktree maintenance".into(),
1620 detail: detail.clone(),
1621 at: Timestamp::now(),
1622 });
1623 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624 }
1625 };
1626 let disk = RunState::load(&self.state.id)?;
1627 crate::pr_head::ensure_owner(&disk)?;
1628 let claimed =
1629 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630 anyhow::ensure!(
1635 disk.liveness(claimed) != crate::run::Liveness::Live,
1636 "run already has a live driver; wait for its execution to end"
1637 );
1638 if let Some(to) = &disk.released_to {
1639 bail!(
1640 "run {} cannot continue: its worktree was released to run {}",
1641 self.state.short(),
1642 crate::run::short_of(to)
1643 );
1644 }
1645 anyhow::ensure!(
1646 serde_json::to_value(&disk.base_sync_recovery)?
1647 == serde_json::to_value(&self.state.base_sync_recovery)?,
1648 "base recovery changed since this run was loaded; reload the saved run"
1649 );
1650 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651 let mut execution = DriverExecution {
1652 runner: self,
1653 driver,
1654 };
1655 let _owner = owner;
1658 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659 .await?;
1660 let state = &mut execution.runner.state;
1661 state.parked = false;
1666 state.resource_wait = None;
1667 state.resource_block = None;
1668 state.clear_active();
1675 let pid = std::process::id();
1690 state.driver_pid = Some(pid);
1691 state.driver_started_at = crate::proc::process_started_at(pid);
1692 state.driver_exited = false;
1693 state.driver_claimed = Some(true);
1694 state.begin_pass();
1695 execution.driver.bind(state);
1696 state.save()?;
1697 execution.driver.record(state)?;
1698 Ok(execution)
1699 }
1700
1701 async fn execute_granted_repair(&mut self) -> Result<()> {
1704 use crate::repair_budget::{Kind, Phase};
1705 let index = self.state.repair_grants.len() - 1;
1706 let winner = self
1707 .state
1708 .winner()
1709 .cloned()
1710 .context("repair winner missing")?;
1711 loop {
1712 let grant = self.state.repair_grants[index].clone();
1713 match grant.phase {
1714 Phase::Pending => {
1715 if grant.spent >= grant.receipt.request.rounds {
1716 self.state.repair_grants[index].phase = Phase::Complete;
1717 self.state.status = RunStatus::Blocked;
1718 self.state.save()?;
1719 return Ok(());
1720 }
1721 self.state.repair_grants[index].spent += 1;
1724 self.state.repair_grants[index].phase = Phase::Fixing;
1725 self.state.save()?;
1726 match grant.receipt.request.kind {
1727 Kind::Gate => {
1728 let failed = self.state.gate.clone();
1729 let _ = self.gate_fix_round(&winner, &failed).await?;
1730 }
1731 Kind::Review => {
1732 let mut record = self
1733 .state
1734 .reviews
1735 .last()
1736 .cloned()
1737 .context("review failure missing")?;
1738 let round = record.round;
1739 let step = self.run_fix_round(&winner, &mut record).await?;
1740 self.state.reviews[round - 1] = record;
1741 if matches!(step, FixStep::Recorded { report_lost: true }) {
1742 self.state.repair_grants[index].phase = Phase::Complete;
1743 self.state.status = RunStatus::Blocked;
1744 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745 self.state.save()?;
1746 return Ok(());
1747 }
1748 if matches!(step, FixStep::Resource) {
1749 self.stop_for_resource()?;
1750 return Ok(());
1751 }
1752 self.state.save()?;
1753 }
1754 }
1755 if self.resource_halt()? {
1756 return Ok(());
1757 }
1758 }
1759 Phase::Fixing => {
1760 let _coordination = self.coordinate(&winner.worktree).await?;
1763 let rescued = git::rescue_commit(
1764 &winner.worktree,
1765 "magi: preserve interrupted granted repair",
1766 )
1767 .await?;
1768 self.state.note_withheld("repair-grant", &rescued.withheld);
1769 self.state
1772 .verification_history
1773 .push(crate::run::VerificationEvidence {
1774 superseded_by: None,
1775 archived_at: Some(Timestamp::now()),
1776 fixes_spent: Some(self.state.gate_fixes.len()),
1777 gate_head: self.state.gate_head.take(),
1778 gate_ran: std::mem::take(&mut self.state.gate_ran),
1779 gate: std::mem::take(&mut self.state.gate),
1780 e2e: self.state.head_e2e.take(),
1781 });
1782 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783 self.state
1784 .verification_history
1785 .push(crate::run::VerificationEvidence {
1786 superseded_by: None,
1787 archived_at: Some(Timestamp::now()),
1788 fixes_spent: Some(self.state.gate_fixes.len()),
1789 gate_head: None,
1790 gate_ran: false,
1791 gate: Vec::new(),
1792 e2e: Some(e2e),
1793 });
1794 }
1795 self.state.repair_grants[index].review_start = self.state.reviews.len();
1796 self.state.repair_grants[index].head =
1797 git::rev_parse(&winner.worktree, "HEAD").await?;
1798 self.state.repair_grants[index].phase = Phase::Reviewing;
1799 self.state.save()?;
1800 }
1801 Phase::Reviewing => {
1802 self.review_loop_pass().await?;
1803 if self.resource_halt()? {
1804 return Ok(());
1805 }
1806 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807 let clean = self.state.reviews.len() > grant.review_start
1808 && self.state.reviews.last().is_some_and(|r| {
1809 r.head == head
1810 && r.clean
1811 && r.expected > 0
1812 && r.answered == r.expected
1813 && matches!(
1814 r.e2e_status(),
1815 E2eStatus::Passed | E2eStatus::NotConfigured
1816 )
1817 });
1818 if !clean {
1819 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821 == Kind::Review
1822 && grant.spent < grant.receipt.request.rounds
1823 && self.state.reviews.last().is_some_and(|r| {
1824 r.fix.is_none()
1825 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826 }) {
1827 Phase::Pending
1828 } else {
1829 Phase::Complete
1830 };
1831 self.state.status = RunStatus::Blocked;
1832 self.state.event(
1833 "repair-grant",
1834 "fresh review did not clear the repaired head; no publication",
1835 );
1836 self.state.save()?;
1837 if self.state.repair_grants[index].phase == Phase::Pending {
1838 continue;
1839 }
1840 return Ok(());
1841 }
1842 self.state.repair_grants[index].phase = Phase::Gating;
1843 self.state.save()?;
1844 }
1845 Phase::Gating => {
1846 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847 self.state.reconcile_verification_head(&head);
1848 let wait = self.repository_wait();
1849 if self
1850 .state
1851 .reviews
1852 .last()
1853 .is_none_or(|r| r.head != head || !r.clean)
1854 {
1855 self.state.status = RunStatus::Blocked;
1856 self.state.repair_grants[index].phase = Phase::Complete;
1857 self.state.event(
1858 "repair-grant",
1859 "head changed since fresh review; repair held",
1860 );
1861 self.state.save()?;
1862 return Ok(());
1863 }
1864 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865 return Ok(());
1866 }
1867 let outcomes = self.run_gate(&winner).await?;
1868 if verify_inconclusive(&outcomes) {
1869 return Ok(());
1870 }
1871 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872 if after != head {
1873 self.state.repair_grants[index].review_start = self.state.reviews.len();
1874 self.state.repair_grants[index].head = after;
1875 self.state.repair_grants[index].phase = Phase::Reviewing;
1876 self.state.status = RunStatus::Blocked;
1877 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878 self.state.save()?;
1879 return Ok(());
1880 }
1881 let passed = outcomes.iter().all(CommandOutcome::ok);
1882 self.state.repair_grants[index].succeeded = passed;
1883 self.state.gate = outcomes;
1884 self.state.gate_ran = true;
1885 self.state.status = if passed {
1886 RunStatus::Gating
1887 } else {
1888 RunStatus::Blocked
1889 };
1890 self.state.repair_grants[index].phase = if !passed
1891 && grant.receipt.request.kind == Kind::Gate
1892 && grant.spent < grant.receipt.request.rounds
1893 {
1894 Phase::Pending
1895 } else {
1896 Phase::Complete
1897 };
1898 if passed && self.state.config.merge.mode == MergeMode::Pr {
1899 self.state.pr_publication_pending = true;
1901 } else if passed {
1902 self.state.status = RunStatus::Ready;
1903 }
1904 self.state.save()?;
1905 if !passed {
1906 continue;
1907 }
1908 if self.state.config.merge.mode != MergeMode::Pr {
1912 self.state.status = RunStatus::Ready;
1913 self.state.event(
1914 "repair-grant",
1915 "repair verified locally; grant carries no merge authorization",
1916 );
1917 self.state.save()?;
1918 return Ok(());
1919 }
1920 return self.merge().await;
1921 }
1922 Phase::Complete => return Ok(()),
1923 }
1924 }
1925 }
1926
1927 async fn execute_graph(&mut self) -> Result<()> {
1928 self.state.configure_recovery_resume()?;
1929 if self
1930 .state
1931 .repair_grants
1932 .last()
1933 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1934 {
1935 return self.execute_granted_repair().await;
1936 }
1937 if self
1938 .state
1939 .repair_grants
1940 .last()
1941 .is_some_and(|g| !g.succeeded)
1942 {
1943 self.state.status = RunStatus::Blocked;
1944 self.state.save()?;
1945 return Ok(());
1946 }
1947 self.persist_candidates().await?;
1950 if self.state.base_sync_blocked() {
1953 self.sync_to_base().await?;
1954 if !self.state.base_sync_blocked()
1955 && self.state.status != RunStatus::AlreadyInBase
1956 && !self.resource_halt()?
1957 {
1958 self.finish_after_tally().await?;
1959 }
1960 return Ok(());
1961 }
1962 if self
1963 .state
1964 .merge_decision
1965 .as_ref()
1966 .is_some_and(|decision| !decision.closed)
1967 && self.state.status != RunStatus::Landing
1968 {
1969 if crate::merge_decision::resumable_hold(&self.state) {
1970 self.state.merge = Some(MergeOutcome {
1971 mode: MergeMode::Pr,
1972 ok: false,
1973 detail: self
1974 .state
1975 .merge_decision
1976 .as_ref()
1977 .unwrap()
1978 .target
1979 .pr
1980 .clone(),
1981 empty: false,
1982 });
1983 self.run_land().await?;
1984 self.settle_questions();
1985 return Ok(());
1986 }
1987 bail!(
1988 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1989 );
1990 }
1991 if self.state.config.graph.selected_implementer.is_some()
1992 && self.state.status == RunStatus::Blocked
1993 && self.state.tally.is_none()
1994 && self.state.candidate_preservation.is_empty()
1995 && self.state.interrupted_implementations.is_empty()
1996 {
1997 bail!(
1998 "selected implementation is held; explicitly release the owning task to retry it"
1999 );
2000 }
2001
2002 if self.state.merge_repairs.last().is_some_and(|r| {
2003 matches!(
2004 r.phase,
2005 crate::run::MergeRepairPhase::Requested
2006 | crate::run::MergeRepairPhase::Fixing
2007 | crate::run::MergeRepairPhase::Reviewing
2008 )
2009 }) {
2010 if self.repair_merge_review().await? {
2011 self.run_land().await?;
2012 }
2013 self.settle_questions();
2014 return Ok(());
2015 }
2016 if self.state.pr_publication_pending {
2019 self.merge().await?;
2020 if self.resource_halt()? {
2021 return Ok(());
2022 }
2023 self.state.save()?;
2024 self.settle_questions();
2025 return Ok(());
2026 }
2027 if self.state.status == RunStatus::Stalled {
2040 if self.recover_stall().await? {
2041 self.finish_after_tally().await?;
2042 } else {
2043 self.state.save()?;
2045 }
2046 return Ok(());
2047 }
2048 if self.state.status == RunStatus::Landing
2058 || crate::handover::landing_checkpoint(&self.state)
2059 {
2060 self.state.land_sync_pending = true;
2061 self.state.save()?;
2062 self.run_land().await?;
2063 self.settle_questions();
2068 return Ok(());
2069 }
2070 if self.state.github_text.is_some()
2075 && self.state.merge.is_none()
2076 && self.state.status == RunStatus::Gating
2077 {
2078 self.merge().await?;
2079 if self.resource_halt()? {
2080 return Ok(());
2081 }
2082 self.state.save()?;
2083 return Ok(());
2084 }
2085 if let Err(e) = self.prep().await {
2086 if e.is::<crate::repo_lock::ResourceBlocked>() {
2087 return Err(e);
2088 }
2089 self.state.status = RunStatus::Blocked;
2090 self.state
2091 .prep_failure
2092 .get_or_insert_with(|| format!("{e:#}"));
2093 self.state
2094 .event("prep", format!("preparation stopped: {e:#}"));
2095 self.state.save()?;
2096 return Err(e);
2097 }
2098 if self.park_here()? {
2099 return Ok(());
2100 }
2101 self.advise().await?;
2102 if self.park_here()? {
2103 return Ok(());
2104 }
2105 self.implement().await?;
2106 if self.park_here()? {
2107 return Ok(());
2108 }
2109 if self.state.status == RunStatus::VerifiedNoop {
2113 return Ok(());
2114 }
2115 self.judge().await?;
2116 if self.park_here()? {
2117 return Ok(());
2118 }
2119 self.deliberate().await?;
2120 if self.park_here()? {
2121 return Ok(());
2122 }
2123 self.vote().await?;
2124 if self.park_here()? {
2125 return Ok(());
2126 }
2127 self.tally()?;
2128 if self.state.status == RunStatus::Stalled {
2133 self.state.save()?;
2137 return Ok(());
2138 }
2139 self.finish_after_tally().await?;
2140 Ok(())
2141 }
2142
2143 fn stop_for_resource(&mut self) -> Result<()> {
2148 if self.pause.parked() || self.interrupt.parked() {
2151 self.park_here()?;
2152 return Ok(());
2153 }
2154 if self.state.resource_block.is_some() {
2155 self.state.status = RunStatus::Blocked;
2156 }
2157 self.state.save()
2158 }
2159
2160 fn resource_halt(&mut self) -> Result<bool> {
2162 if self.state.resource_block.is_none() {
2163 return Ok(false);
2164 }
2165 self.stop_for_resource()?;
2166 Ok(true)
2167 }
2168
2169 fn park_here(&mut self) -> Result<bool> {
2176 let parking = self.pause.parked() || self.interrupt.parked();
2177 if !parking && self.state.resource_block.is_some() {
2178 self.stop_for_resource()?;
2179 return Ok(true);
2180 }
2181 if !self.pause.parked() && !self.interrupt.parked() {
2186 return Ok(false);
2187 }
2188 if self.state.parked {
2189 return Ok(true);
2190 }
2191 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2192 if let Some(block) = &mut self.state.resource_block
2193 && matches!(
2194 block.cause,
2195 crate::run::ResourceBlockCause::Cancelled { .. }
2196 )
2197 {
2198 block.cause = crate::run::ResourceBlockCause::Cancelled {
2199 park: cause.clone(),
2200 };
2201 }
2202 let why = match cause.as_ref().map(ParkCause::label) {
2203 Some(reason) => format!(
2204 "parked after `{}` ({reason}) — resume to carry on from here",
2205 self.state.status.as_str()
2206 ),
2207 None => format!(
2208 "parked after `{}` — resume to carry on from here",
2209 self.state.status.as_str()
2210 ),
2211 };
2212 self.state.event("park", why);
2213 self.state
2214 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2215 self.state.parked = true;
2216 self.state.save()?;
2217 Ok(true)
2218 }
2219
2220 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2222 if self.state.prep_failure.is_none() {
2223 return Ok(());
2224 }
2225 let failed_at = self
2226 .state
2227 .events
2228 .iter()
2229 .rev()
2230 .find(|event| event.node == "prep")
2231 .map_or(self.state.updated_at, |event| event.at);
2232 anyhow::ensure!(
2233 approval_at > failed_at,
2234 "preparation retry requires a new explicit release after the failure"
2235 );
2236 self.state.prep_failure = None;
2237 self.state.prep_complete = Some(false);
2238 self.state
2239 .event("prep-retry", "operator approved retry of preparation");
2240 Ok(())
2241 }
2242
2243 pub fn on_pause(&mut self, pause: Pause) {
2245 self.pause = pause;
2246 }
2247
2248 pub fn watch_interrupt(&mut self, pause: Pause) {
2254 self.interrupt = pause;
2255 }
2256
2257 fn settle_questions(&mut self) {
2277 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2278 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2279 }
2280 }
2281
2282 async fn finish_after_tally(&mut self) -> Result<()> {
2285 self.graft().await?;
2288 if self.resource_halt()? {
2289 return Ok(());
2290 }
2291 self.fold_losers().await?;
2292 self.sync_to_base().await?;
2297 if self.hold_base_sync().await? || self.resource_halt()? {
2298 return Ok(());
2299 }
2300 if self.state.status == RunStatus::AlreadyInBase {
2301 return Ok(());
2302 }
2303 self.review_loop().await?;
2304 if self.resource_halt()? {
2305 return Ok(());
2306 }
2307 self.sync_to_base().await?;
2308 if self.hold_base_sync().await? || self.resource_halt()? {
2309 return Ok(());
2310 }
2311 if self.state.status == RunStatus::AlreadyInBase {
2312 return Ok(());
2313 }
2314 self.gate().await?;
2315 self.merge().await?;
2316 if self.resource_halt()? {
2317 return Ok(());
2318 }
2319 self.state.save()?;
2320 Ok(())
2321 }
2322
2323 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2330 let result = match ensure_setup_config(&mut self.state, repo).await {
2331 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2332 Err(e) => Err(e),
2333 };
2334 if let Err(e) = result {
2335 git::worktree_remove_clean(repo, wt).await.ok();
2336 if in_prep {
2337 self.rollback_prep(repo).await;
2338 }
2339 self.state.status = RunStatus::Blocked;
2340 self.state.event(
2341 "setup",
2342 format!("worktree setup failed in {}: {e:#}", wt.display()),
2343 );
2344 self.state.save()?;
2345 return Err(e);
2346 }
2347 Ok(())
2348 }
2349
2350 async fn rollback_prep(&mut self, repo: &Path) {
2352 let root = self.state.worktree_root();
2353 for c in std::mem::take(&mut self.state.candidates) {
2354 if matches!(
2355 git::worktree_remove_clean(repo, &c.worktree).await,
2356 Ok(true)
2357 ) {
2358 git::branch_delete(repo, &c.branch).await.ok();
2359 } else {
2360 self.state.candidates.push(c);
2361 }
2362 }
2363 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2364 for j in 1..=seats {
2365 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2366 .await
2367 .ok();
2368 }
2369 for k in 1..=self.state.config.graph.advisors {
2370 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2371 .await
2372 .ok();
2373 }
2374 }
2375
2376 async fn prep(&mut self) -> Result<()> {
2377 if let Some(reason) = &self.state.prep_failure {
2378 bail!(
2379 "candidate preparation previously failed: {reason}; correct the source before retrying"
2380 );
2381 }
2382 if self
2383 .state
2384 .prep_complete
2385 .unwrap_or(!self.state.candidates.is_empty())
2386 {
2387 return Ok(());
2388 }
2389 let repo = self.state.repo.clone();
2390 let _coordination = self.coordinate(&repo).await?;
2393 let base = self.state.base_commit.clone();
2394 let plan = self.state.sources.plan(&repo, &base).await?;
2395 self.rollback_prep(&repo).await;
2396 if !self.state.candidates.is_empty() {
2397 bail!(
2398 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2399 );
2400 }
2401 self.state.prep_complete = Some(false);
2402 self.state.status = RunStatus::Prep;
2403 self.state.save()?;
2404 let repo = self.state.repo.clone();
2405 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2406 let root = self.state.worktree_root();
2407 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2408
2409 let hooks_dir = self.state.dir().join("hooks");
2412 if self.state.config.blind.commit_msg_hook {
2413 std::fs::create_dir_all(&hooks_dir)
2414 .with_context(|| format!("create {}", hooks_dir.display()))?;
2415 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2416 let path = hooks_dir.join("commit-msg");
2417 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2418 make_executable(&path)?;
2419 git::acquire_worktree_config(&repo).await?;
2427 self.state.enabled_worktree_config = true;
2428 }
2429
2430 ensure_setup_config(&mut self.state, &repo).await?;
2433
2434 for (index, (spec, label)) in self
2435 .roles
2436 .implementers
2437 .clone()
2438 .into_iter()
2439 .zip(labels)
2440 .enumerate()
2441 {
2442 let branch = self.state.branch_for(label);
2443 let worktree = root.join(format!("cand-{label}"));
2444 self.state.event(
2448 "prep-intent",
2449 serde_json::json!({
2450 "branch": branch,
2451 "worktree": worktree,
2452 "start": start,
2453 "picks": plan.picks,
2454 })
2455 .to_string(),
2456 );
2457 self.state.save()?;
2458 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2459 if self.state.config.blind.commit_msg_hook {
2460 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2461 }
2462 git::local_exclude(&worktree, "/.magi/").await?;
2463 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2464 git::worktree_remove_clean(&repo, &worktree).await.ok();
2467 git::branch_delete(&repo, &branch).await.ok();
2468 self.rollback_prep(&repo).await;
2469 self.state.status = RunStatus::Blocked;
2470 self.state
2471 .event("prep", format!("worktree setup failed: {e:#}"));
2472 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2473 self.state.save()?;
2474 return Err(e);
2475 }
2476 self.state.candidates.push(Candidate {
2477 index,
2478 label,
2479 agent: spec.id.clone(),
2480 branch,
2481 worktree,
2482 summary: String::new(),
2483 stat: String::new(),
2484 files: 0,
2485 commits: 0,
2486 empty: false,
2487 failed: None,
2488 verified_noop: None,
2489 duration_ms: 0,
2490 folded: false,
2491 });
2492 self.state.save()?;
2493 for pick in &plan.picks {
2494 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2495 {
2496 self.state.status = RunStatus::Blocked;
2497 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2498 self.state
2499 .event("prep", format!("cannot apply referenced commit: {e}"));
2500 self.state.save()?;
2501 return Err(e);
2502 }
2503 }
2504 }
2505
2506 for j in 1..=self.roles.judges.len() {
2507 let wt = root.join(format!("judge-{j}"));
2508 if !wt.exists() {
2509 git::worktree_add_detached(&repo, &wt, &base).await?;
2510 self.setup_seat_worktree(&repo, &wt, true).await?;
2511 }
2512 }
2513
2514 if self.state.config.graph.advise {
2522 for k in 1..=self.state.config.graph.advisors {
2523 let wt = root.join(format!("advisor-{k}"));
2524 if !wt.exists() {
2525 git::worktree_add_detached(&repo, &wt, &base).await?;
2526 self.setup_seat_worktree(&repo, &wt, true).await?;
2527 }
2528 }
2529 }
2530
2531 let authors: Vec<&str> = self
2536 .roles
2537 .implementers
2538 .iter()
2539 .map(|a| a.id.as_str())
2540 .collect();
2541 let overlap: Vec<String> = self
2542 .roles
2543 .judges
2544 .iter()
2545 .enumerate()
2546 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2547 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2548 .collect();
2549 if !overlap.is_empty() {
2550 let note = format!(
2551 "{} also authored a candidate; blind, but the panel is less \
2552 independent than {} distinct agents would be",
2553 overlap.join(", "),
2554 self.roles.judges.len()
2555 );
2556 self.state.event("prep", note);
2557 }
2558
2559 self.state.event(
2560 "prep",
2561 format!(
2562 "{} candidates, {} judges, base {} ({})",
2563 self.state.candidates.len(),
2564 self.roles.judges.len(),
2565 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2566 self.state.base_branch
2567 ),
2568 );
2569 self.state.prep_complete = Some(true);
2570 self.state.status = RunStatus::Implementing;
2571 self.state.save()?;
2572 Ok(())
2573 }
2574
2575 async fn advise(&mut self) -> Result<()> {
2608 let implement_untouched = self
2609 .state
2610 .candidates
2611 .iter()
2612 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2613 if !self.state.config.graph.advise || self.state.advise_attempted {
2614 return Ok(());
2615 }
2616 if !implement_untouched {
2617 self.state.event(
2618 "advise",
2619 "skipping the design-deliberation stage: at least one \
2620 candidate already shows implementation progress, so this \
2621 run is past the point the stage exists to run before"
2622 .to_owned(),
2623 );
2624 self.state.advise_attempted = true;
2625 self.state.save()?;
2626 return Ok(());
2627 }
2628 let run_id = self.state.id.clone();
2629 let prompts = self.state.config.prompts.clone();
2630 let instruction = self.state.instruction.clone();
2631 let language = self.state.config.graph.language.clone();
2632 let root = self.state.worktree_root();
2633 let n = self.state.config.graph.advisors;
2634 let where_recorded = self.state.dir().join("run.json");
2635
2636 let seats = match self.state.config.advisors() {
2637 Ok(seats) if !seats.is_empty() => seats,
2638 Ok(_) => {
2639 self.state.event(
2640 "advise",
2641 format!(
2642 "[graph] advisors is 0; skipping the design-deliberation \
2643 stage and continuing without a synthesis brief (see {})",
2644 where_recorded.display()
2645 ),
2646 );
2647 self.state.advise_attempted = true;
2648 self.state.save()?;
2649 return Ok(());
2650 }
2651 Err(e) => {
2652 self.state.event(
2653 "advise",
2654 format!(
2655 "could not resolve advisor seats ({e:#}); continuing \
2656 without a design-deliberation brief (see {})",
2657 where_recorded.display()
2658 ),
2659 );
2660 self.state.advise_attempted = true;
2661 self.state.save()?;
2662 return Ok(());
2663 }
2664 };
2665
2666 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2667 let artifacts = agent::artifacts_dir(&self.state.dir());
2668 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2669
2670 let mut jobs = Vec::new();
2671 for (i, spec) in seats.iter().cloned().enumerate() {
2672 let seat_key = format!("advisor-{}", i + 1);
2673 let seat = self.seat(&seat_key, &spec.id);
2674 jobs.push(SeatJob {
2675 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2676 spec,
2677 seat,
2678 cwd: worktrees[i % worktrees.len()].clone(),
2679 timeout,
2680 allow_write: false,
2681 sessions: false,
2682 artifacts: artifacts.clone(),
2683 stem: seat_key,
2684 handover: None,
2685 });
2686 }
2687
2688 self.state.event(
2689 "advise",
2690 format!(
2691 "{} advisor seat(s) sketching a design in parallel",
2692 jobs.len()
2693 ),
2694 );
2695 let mut quota_losses = Vec::new();
2696 let cache = self.state.config.cache_dir();
2697 let ctx = WaveCtx {
2698 carry_seats: false,
2699 pauses: Some((&self.pause, &self.interrupt)),
2700 run: &run_id,
2701 node: "advise",
2702 prompts: &prompts,
2703 cache: cache.as_deref(),
2704 round: None,
2705 };
2706 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2707 let results = ask_json_wave::<Proposal>(
2708 jobs,
2709 Arc::clone(&self.sem),
2710 self.state.config.graph.retries,
2711 &advisor_roster,
2712 &ctx,
2713 &mut quota_losses,
2714 &mut self.state,
2715 &|p: &Proposal| p.validate(),
2716 )
2717 .await;
2718 if self.state.resource_block.is_some() {
2719 return self.stop_for_resource();
2720 }
2721 self.state.quota.extend(quota_losses);
2722
2723 let mut records = Vec::with_capacity(results.len());
2724 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2725 let agent_id = seat.agent.clone();
2726 self.state.seats.insert(seat.key.clone(), seat);
2727 match res {
2728 Ok((proposal, out)) => {
2729 self.state
2730 .event("advise", format!("advisor-{} proposed a design", i + 1));
2731 records.push(advise::AdvisorRecord::proposed(
2732 i + 1,
2733 agent_id,
2734 proposal,
2735 out.duration_ms,
2736 ));
2737 }
2738 Err(e) => {
2739 self.state.event(
2740 "advise",
2741 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2742 );
2743 records.push(advise::AdvisorRecord::failed(
2744 i + 1,
2745 agent_id,
2746 e.to_string(),
2747 ));
2748 }
2749 }
2750 }
2751
2752 let mut advice = advise::Advice {
2753 records,
2754 synthesis: None,
2755 };
2756 if advice.proposals().is_empty() {
2757 self.state.event(
2758 "advise",
2759 "no advisor produced a usable proposal; continuing without a \
2760 synthesis brief"
2761 .to_owned(),
2762 );
2763 } else {
2764 match self
2765 .synthesize_brief(
2766 &advice,
2767 &instruction,
2768 &language,
2769 &worktrees[0],
2770 &artifacts,
2771 &run_id,
2772 &prompts,
2773 cache.as_deref(),
2774 )
2775 .await
2776 {
2777 Ok(Some(text)) => {
2778 self.state.event(
2779 "advise",
2780 "synthesized a design brief for the implementer".to_owned(),
2781 );
2782 advice.synthesis = Some(text);
2783 }
2784 Ok(None) => {
2785 self.state.event(
2786 "advise",
2787 "the synthesis seat produced nothing usable; continuing \
2788 without a design brief"
2789 .to_owned(),
2790 );
2791 }
2792 Err(e) => {
2793 if e.is::<crate::repo_lock::ResourceBlocked>() {
2794 return Err(e);
2795 }
2796 self.state.event(
2797 "advise",
2798 format!("could not synthesize a design brief: {e:#}"),
2799 );
2800 }
2801 }
2802 }
2803 advise::apply_reflection(&mut advice);
2804
2805 self.state.advice = Some(advice);
2806 self.state.advise_attempted = true;
2807 self.state.save()?;
2808 Ok(())
2809 }
2810
2811 #[allow(clippy::too_many_arguments)]
2823 async fn synthesize_brief(
2824 &mut self,
2825 advice: &advise::Advice,
2826 instruction: &str,
2827 language: &str,
2828 cwd: &Path,
2829 artifacts: &Path,
2830 run_id: &str,
2831 prompts: &Prompts,
2832 cache: Option<&Path>,
2833 ) -> Result<Option<String>> {
2834 let chain = agent::pick_chain(
2835 &self.state.config.agents,
2836 self.state.config.roles.synthesizer.as_ref(),
2837 &agent::installed,
2838 "synthesizer",
2839 )?;
2840 let proposals = advice.proposals();
2841 let mut prompt = prompt::with_overlay(
2842 prompt::synthesize_brief(instruction, &proposals, language),
2843 prompts.overlay("advise"),
2844 );
2845 if cache.is_some() {
2846 prompt.push('\n');
2851 prompt.push_str(&prompt::build_cache_note("advise", false));
2852 }
2853 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2854 let mut last = None;
2859 for (n, spec) in chain.iter().enumerate() {
2860 if n > 0 {
2861 self.state
2862 .event("advise", format!("synthesis falling back to {}", spec.id));
2863 }
2864 let mut seat = self.seat("advise-synthesis", &spec.id);
2865 let wait = self.repository_wait();
2866 let outcome = wait
2867 .scope(agent::invoke(
2868 spec,
2869 &mut seat,
2870 &Invocation {
2871 coordination: self.state.config.coordination.as_ref(),
2872 fetched_base: None,
2873 cwd,
2874 prompt: &prompt,
2875 timeout,
2876 allow_write: false,
2877 sessions: false,
2878 artifacts,
2879 stem: &if n == 0 {
2880 "advise-synthesis".to_owned()
2881 } else {
2882 format!("advise-synthesis-{}", spec.id)
2883 },
2884 run: run_id,
2885 node: "advise",
2886 cache_dir: None,
2887 attachments: &[],
2888 writable: &[],
2889 },
2890 ))
2891 .await;
2892 if outcome.is_ok() {
2893 self.state.seats.insert(seat.key.clone(), seat);
2894 }
2895 let advance = agent::chain_advances(&outcome);
2896 last = Some(outcome);
2897 if !advance {
2898 break;
2899 }
2900 }
2901 let out = last.expect("a chain holds at least one agent")?;
2904 if !out.usable() {
2905 return Ok(None);
2906 }
2907 let text =
2908 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2909 Ok((!text.trim().is_empty()).then_some(text))
2910 }
2911
2912 async fn implement(&mut self) -> Result<()> {
2915 self.persist_candidates().await?;
2916 let run_id = self.state.id.clone();
2921 let prompts = self.state.config.prompts.clone();
2922 let todo: Vec<usize> = self
2923 .state
2924 .candidates
2925 .iter()
2926 .enumerate()
2927 .filter(|(_, c)| {
2928 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2929 && c.failed.is_none()
2930 && !c.empty
2931 })
2932 .map(|(i, _)| i)
2933 .collect();
2934 if todo.is_empty() {
2935 return self.after_implement();
2936 }
2937 self.state.status = RunStatus::Implementing;
2938
2939 let language = self.state.config.graph.language.clone();
2940 let github_language = self.state.config.graph.github_language.clone();
2941 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2942 let sessions = self.state.config.graph.sessions;
2943 let artifacts = agent::artifacts_dir(&self.state.dir());
2944 let brief = self
2948 .state
2949 .advice
2950 .as_ref()
2951 .and_then(|a| a.synthesis.as_deref())
2952 .map(str::to_owned);
2953 let attachments = self.state.attachments.clone();
2954
2955 let mut jobs = Vec::new();
2956 for &i in &todo {
2957 let (index, label, worktree) = {
2958 let c = &self.state.candidates[i];
2959 (c.index, c.label, c.worktree.clone())
2960 };
2961 let spec = self.roles.implementers[index].clone();
2962 let seat_key = format!("impl-{label}");
2963 let seat = self.seat(&seat_key, &spec.id);
2964 let instruction = seeded_instruction(&self.state);
2965 jobs.push(SeatJob {
2966 spec,
2967 seat,
2968 prompt: prompt::implement_in(
2969 &instruction,
2970 &worktree.to_string_lossy(),
2971 &language,
2972 &github_language,
2973 brief.as_deref(),
2974 &attachments,
2975 ),
2976 cwd: worktree,
2977 timeout,
2978 allow_write: true,
2979 sessions,
2980 artifacts: artifacts.clone(),
2981 stem: format!("impl-{label}"),
2982 handover: None,
2983 });
2984 }
2985
2986 self.state.event(
2987 "implement",
2988 format!("{} candidates in parallel", jobs.len()),
2989 );
2990 let mut sent = jobs.clone();
2996 let cache = self.state.config.cache_dir();
2997 let ctx = WaveCtx {
2998 carry_seats: false,
2999 pauses: Some((&self.pause, &self.interrupt)),
3000 run: &run_id,
3001 node: "implement",
3002 prompts: &prompts,
3003 cache: cache.as_deref(),
3004 round: None,
3005 };
3006 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3007 if !wave_blocked(&results) {
3008 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3009 .await;
3010 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3011 .await?;
3012 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3013 .await;
3014 }
3015 if !results.is_empty()
3018 && results
3019 .iter()
3020 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3021 {
3022 let losses: Vec<_> = results
3023 .iter()
3024 .map(|(_, seat, out)| {
3025 let AgentOutcome::Quota(o) = out else {
3026 unreachable!()
3027 };
3028 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3029 })
3030 .collect();
3031 self.state.quota.extend(losses.clone());
3032 for (_, seat, _) in results {
3033 self.state.seats.insert(seat.key.clone(), seat);
3034 }
3035 for &i in &todo {
3036 let label = self.state.candidates[i].label;
3037 self.state.interrupted_implementations.insert(label);
3038 self.state
3039 .candidate_preservation
3040 .insert(label, "machine outage awaiting persistence".into());
3041 }
3042 block_machine_outage(&mut self.state, "implement", &losses);
3045 self.state.save()?;
3046 self.persist_candidates().await?;
3047 for &i in &todo {
3048 self.state
3049 .interrupted_implementations
3050 .insert(self.state.candidates[i].label);
3051 }
3052 return self.stop_for_resource();
3053 }
3054 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3055
3056 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3057 && results
3058 .iter()
3059 .any(|(_, _, out)| FailClass::of(out).is_some());
3060 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3061 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3062 if self
3066 .state
3067 .candidate_preservation
3068 .contains_key(&self.state.candidates[i].label)
3069 {
3070 self.state.seats.insert(seat.key.clone(), seat);
3071 }
3072 continue;
3073 }
3074 let seat_key = seat.key.clone();
3075 let agent = seat.agent.clone();
3084 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3085 self.state.seats.insert(seat.key.clone(), seat);
3086 let label = self.state.candidates[i].label;
3087 self.state.interrupted_implementations.remove(&label);
3088
3089 let (summary, duration, failed, verified_claim) = match out {
3090 AgentOutcome::Ok(o) => {
3091 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3092 let failed = (!o.usable()).then(|| {
3093 if o.timed_out {
3094 "agent timed out".to_owned()
3095 } else {
3096 format!(
3097 "agent exited with {}",
3098 crate::notices::exit_status(o.exit_code)
3099 )
3100 }
3101 });
3102 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3103 (text, o.duration_ms, failed, verified_claim)
3104 }
3105 AgentOutcome::Dropped(o) => {
3111 let why = o
3112 .dropped
3113 .as_ref()
3114 .map(|d| d.why.as_str())
3115 .unwrap_or("the CLI ended the stream without delivering its answer");
3116 (
3117 String::new(),
3118 o.duration_ms,
3119 Some(format!("the CLI dropped the stream ({why})")),
3120 None,
3121 )
3122 }
3123 AgentOutcome::Quota(o) => {
3124 self.state.quota.push(QuotaLoss::from_report(
3125 seat_key,
3126 "implement",
3127 o.quota.as_ref(),
3128 ));
3129 (
3130 String::new(),
3131 o.duration_ms,
3132 Some("rate limited (quota); produced no change".to_owned()),
3133 None,
3134 )
3135 }
3136 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3137 (String::new(), 0, Some(e), None)
3138 }
3139 };
3140
3141 let c = &mut self.state.candidates[i];
3144 if !exhausted_the_fallback_chain {
3145 c.agent = agent;
3146 }
3147 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3148 c.duration_ms = duration;
3149 c.failed = failed;
3150 c.verified_noop = verified_claim;
3151 self.state
3152 .candidate_preservation
3153 .insert(label, "awaiting persistence".to_owned());
3154 }
3155 self.state.save()?;
3156 self.persist_candidates().await?;
3157
3158 if resource_blocked {
3159 return self.stop_for_resource();
3160 }
3161 if selected_failed {
3162 self.state.status = RunStatus::Blocked;
3163 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3164 self.state.save()?;
3165 bail!(
3166 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3167 );
3168 }
3169 self.after_implement()
3170 }
3171
3172 async fn persist_candidates(&mut self) -> Result<()> {
3174 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3175 for label in labels {
3176 let i = self
3177 .state
3178 .candidates
3179 .iter()
3180 .position(|c| c.label == label)
3181 .context("pending candidate is missing")?;
3182 let c = self.state.candidates[i].clone();
3183 let interrupted = self.state.interrupted_implementations.contains(&label);
3184 let result = async {
3185 let wait = crate::repo_lock::WaitPolicy::new(
3186 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3187 || false,
3188 );
3189 let _coordination =
3190 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3191 if let Err(e) = git::snapshot_uncommitted(
3194 &c.worktree,
3195 &self
3196 .state
3197 .dir()
3198 .join("artifacts")
3199 .join(format!("candidate-{label}-recovery")),
3200 )
3201 .await
3202 {
3203 self.state.event(
3204 "preserve",
3205 format!("candidate {label} snapshot incomplete: {e:#}"),
3206 );
3207 }
3208 let rescued = git::rescue_commit(
3209 &c.worktree,
3210 &format!("magi: candidate {label} (uncommitted work)"),
3211 )
3212 .await?;
3213 self.state.note_withheld("implement", &rescued.withheld);
3214 let base = &self.state.base_commit;
3215 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3216 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3217 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3218 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3219 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3220 if c.empty && commits > 0 && !patch.trim().is_empty()
3221 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3222 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3223 || self.state.tally.is_some())
3224 {
3225 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3229 }
3230 let c = &mut self.state.candidates[i];
3231 c.commits = commits;
3232 c.stat = stat;
3233 c.files = files;
3234 let no_changes = commits == 0 || patch.trim().is_empty();
3235 c.empty = no_changes && !interrupted;
3236 if interrupted && no_changes {
3237 c.failed = None;
3238 c.verified_noop = None;
3239 }
3240 if !c.empty {
3241 c.failed = None;
3242 c.verified_noop = None;
3243 }
3244 self.state.event(
3245 "preserve",
3246 format!("candidate {label}: {files} files, {commits} commits"),
3247 );
3248 Ok::<_, anyhow::Error>(())
3249 }
3250 .await;
3251 match result {
3252 Ok(()) => {
3253 self.state.candidate_preservation.remove(&label);
3254 if !interrupted || self.state.candidates[i].files > 0 {
3255 self.state.interrupted_implementations.remove(&label);
3256 }
3257 }
3258 Err(e) => {
3259 let reason = format!("candidate {label} preservation failed: {e:#}");
3260 self.state.event("preserve", &reason);
3261 self.state.candidate_preservation.insert(label, reason);
3262 }
3263 }
3264 if self.state.candidate_preservation.is_empty()
3265 && self.state.status == RunStatus::Blocked
3266 && self.state.candidates.iter().all(|c| c.failed.is_none())
3267 {
3268 self.state.status = RunStatus::Implementing;
3272 self.state.event(
3273 "preserve",
3274 "candidate persistence recovered; implementation can continue",
3275 );
3276 }
3277 self.state.save()?;
3278 }
3279 if !self.state.candidate_preservation.is_empty() {
3280 self.state.status = RunStatus::Blocked;
3281 self.state.save()?;
3282 bail!(
3283 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3284 );
3285 }
3286 Ok(())
3287 }
3288
3289 async fn resume_undelivered(
3317 &mut self,
3318 results: &mut [(usize, SeatState, AgentOutcome)],
3319 sent: &[SeatJob],
3320 prompts: &Prompts,
3321 run_id: &str,
3322 ) {
3323 for (wi, seat, out) in results.iter_mut() {
3324 let Some(dropped) = (match &*out {
3325 AgentOutcome::Dropped(o) => o.dropped.clone(),
3326 _ => None,
3327 }) else {
3328 continue;
3329 };
3330 let Some(job) = sent.get(*wi) else { continue };
3331 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3333 self.state.event(
3334 "implement",
3335 format!(
3336 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3337 work is in the tree",
3338 seat.key, dropped.output_tokens, dropped.why
3339 ),
3340 );
3341 continue;
3342 }
3343 if !has_context(&job.spec, seat, job.sessions) {
3351 self.state.event(
3352 "implement",
3353 format!(
3354 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3355 is no session left to resume",
3356 seat.key, dropped.output_tokens, dropped.why
3357 ),
3358 );
3359 continue;
3360 }
3361 self.state.event(
3362 "implement",
3363 format!(
3364 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3365 conversation",
3366 seat.key, dropped.output_tokens, dropped.why
3367 ),
3368 );
3369 let mut retry = job.clone();
3370 retry.seat = seat.clone();
3371 retry.prompt = prompt::resume_after_drop(&dropped.why);
3372 retry.timeout = retry_budget(job.timeout, true);
3373 retry.stem = format!("{}-resume", job.stem);
3374 let cache = self.state.config.cache_dir();
3375 let ctx = WaveCtx {
3376 carry_seats: false,
3377 pauses: Some((&self.pause, &self.interrupt)),
3378 run: run_id,
3379 node: "implement",
3380 prompts,
3381 cache: cache.as_deref(),
3382 round: None,
3383 };
3384 let (resumed_seat, resumed) =
3385 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3386 *seat = resumed_seat;
3387 *out = resumed;
3388 }
3389 }
3390
3391 async fn resume_seat_handovers(
3456 &mut self,
3457 results: &mut [(usize, SeatState, AgentOutcome)],
3458 sent: &mut [SeatJob],
3459 prompts: &Prompts,
3460 run_id: &str,
3461 ) -> Result<()> {
3462 let instruction = seeded_instruction(&self.state);
3463 let language = self.state.config.graph.language.clone();
3464 let github_language = self.state.config.graph.github_language.clone();
3465 let brief = self
3466 .state
3467 .advice
3468 .as_ref()
3469 .and_then(|a| a.synthesis.as_deref())
3470 .map(str::to_owned);
3471 let attachments = self.state.attachments.clone();
3472 for (wi, seat, out) in results.iter_mut() {
3473 let others: BTreeSet<String> = sent
3476 .iter()
3477 .enumerate()
3478 .filter(|(j, _)| j != wi)
3479 .map(|(_, j)| j.spec.id.clone())
3480 .collect();
3481 let Some(job) = sent.get_mut(*wi) else {
3482 continue;
3483 };
3484 let start = self
3489 .roles
3490 .implementer_roster
3491 .iter()
3492 .position(|s| s.id == job.spec.id)
3493 .unwrap_or(0);
3494 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3495 let mut fallback_attempt = 0usize;
3496 let mut prev: Option<FailClass> = None;
3497 while let Some(cur) = FailClass::of(&*out) {
3498 if !should_hand_over(prev.as_ref(), &cur) {
3499 break;
3500 }
3501 let Some(next) =
3502 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3503 .cloned()
3504 else {
3505 break;
3506 };
3507 tried.insert(next.id.clone());
3508 fallback_attempt += 1;
3509
3510 let coordination = self.coordinate(&job.cwd).await?;
3511 if let Ok(r) = git::rescue_commit(
3512 &job.cwd,
3513 &format!(
3514 "magi: candidate {} (uncommitted work before {} fallback)",
3515 seat.key,
3516 if cur == FailClass::Quota {
3517 "quota"
3518 } else {
3519 "handover"
3520 }
3521 ),
3522 )
3523 .await
3524 {
3525 self.state.note_withheld("implement", &r.withheld);
3526 }
3527 drop(coordination);
3528
3529 record_handover(
3530 &mut self.state,
3531 "implement",
3532 &seat.key,
3533 &seat.agent,
3534 &next.id,
3535 &cur,
3536 &fail_reason(&*out),
3537 );
3538 prev = Some(cur.clone());
3539
3540 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3541 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3542 job.spec = next.clone();
3550 let mut retry = job.clone();
3551 retry.seat = new_seat;
3552 retry.prompt = prompt::implement_in(
3553 &instruction,
3554 &job.cwd.to_string_lossy(),
3555 &language,
3556 &github_language,
3557 brief.as_deref(),
3558 &attachments,
3559 );
3560 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3561 let cache = self.state.config.cache_dir();
3562 let ctx = WaveCtx {
3563 carry_seats: false,
3564 pauses: Some((&self.pause, &self.interrupt)),
3565 run: run_id,
3566 node: "implement",
3567 prompts,
3568 cache: cache.as_deref(),
3569 round: None,
3570 };
3571 let (fallback_seat, fallback_out) = run_one(
3572 retry,
3573 Arc::clone(&self.sem),
3574 &ctx,
3575 &mut self.state,
3576 fallback_attempt,
3577 )
3578 .await;
3579 *seat = fallback_seat;
3580 *out = fallback_out;
3581 }
3582 }
3583 Ok(())
3584 }
3585
3586 async fn resume_unconfirmed_commands(
3610 &mut self,
3611 results: &mut [(usize, SeatState, AgentOutcome)],
3612 sent: &[SeatJob],
3613 prompts: &Prompts,
3614 run_id: &str,
3615 ) {
3616 for (wi, seat, out) in results.iter_mut() {
3617 let AgentOutcome::Ok(o) = &*out else {
3618 continue;
3619 };
3620 if !has_unconfirmed_command(&o.commands) {
3621 continue;
3622 }
3623 let Some(job) = sent.get(*wi) else { continue };
3624 if !has_context(&job.spec, seat, job.sessions) {
3625 self.state.event(
3626 "implement",
3627 format!(
3628 "{}: the reply named a command whose own CLI never confirmed the exit \
3629 status of, but there is no session left to resume",
3630 seat.key
3631 ),
3632 );
3633 continue;
3634 }
3635 self.state.event(
3636 "implement",
3637 format!(
3638 "{}: the reply named a command whose own CLI never confirmed the exit \
3639 status of; resuming the conversation",
3640 seat.key
3641 ),
3642 );
3643 let mut retry = job.clone();
3644 retry.seat = seat.clone();
3645 retry.prompt = prompt::resume_incomplete(
3646 "a command in your last reply had no confirmed exit status",
3647 );
3648 retry.timeout = retry_budget(job.timeout, true);
3649 retry.stem = format!("{}-confirm", job.stem);
3650 let cache = self.state.config.cache_dir();
3651 let ctx = WaveCtx {
3652 carry_seats: false,
3653 pauses: Some((&self.pause, &self.interrupt)),
3654 run: run_id,
3655 node: "implement",
3656 prompts,
3657 cache: cache.as_deref(),
3658 round: None,
3659 };
3660 let (resumed_seat, resumed) =
3661 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3662 *seat = resumed_seat;
3663 *out = resumed;
3664 }
3665 }
3666
3667 async fn continue_fix_report(
3688 &mut self,
3689 mut seat: SeatState,
3690 parse_err: String,
3691 job: &SeatJob,
3692 prompts: &Prompts,
3693 run_id: &str,
3694 round: usize,
3695 ) -> (
3696 SeatState,
3697 Option<FixReport>,
3698 Option<String>,
3699 ContinuationRecord,
3700 ) {
3701 let mut last_err = parse_err;
3702 let mut cumulative_wait_ms = 0u64;
3703 let mut attempts = 0usize;
3704 loop {
3705 if !has_context(&job.spec, &seat, job.sessions) {
3706 self.state.event(
3707 "fix",
3708 format!(
3709 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3710 session left to resume into"
3711 ),
3712 );
3713 let outcome = if attempts == 0 {
3714 ContinuationOutcome::NoSession
3715 } else {
3716 ContinuationOutcome::Exhausted
3717 };
3718 return (
3719 seat,
3720 None,
3721 Some(format!("unparsable fix report: {last_err}")),
3722 ContinuationRecord {
3723 attempts,
3724 cumulative_wait_ms,
3725 outcome,
3726 },
3727 );
3728 }
3729 if attempts >= MAX_FIX_CONTINUATIONS {
3730 self.state.event(
3731 "fix",
3732 format!(
3733 "round {round}: fixer's reply still had no adoption report after \
3734 {attempts} continuation(s) ({last_err}); giving up"
3735 ),
3736 );
3737 return (
3738 seat,
3739 None,
3740 Some(format!(
3741 "unparsable fix report after {attempts} continuation(s): {last_err}"
3742 )),
3743 ContinuationRecord {
3744 attempts,
3745 cumulative_wait_ms,
3746 outcome: ContinuationOutcome::Exhausted,
3747 },
3748 );
3749 }
3750 attempts += 1;
3751 self.state.event(
3752 "fix",
3753 format!(
3754 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3755 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3756 ),
3757 );
3758 let mut retry = job.clone();
3759 retry.seat = seat.clone();
3760 retry.prompt = prompt::resume_incomplete(&last_err);
3761 retry.timeout = retry_budget(job.timeout, true);
3762 retry.stem = format!("{}-continue{attempts}", job.stem);
3763 let cache = self.state.config.cache_dir();
3764 let ctx = WaveCtx {
3765 carry_seats: false,
3766 pauses: Some((&self.pause, &self.interrupt)),
3767 run: run_id,
3768 node: "fix",
3769 prompts,
3770 cache: cache.as_deref(),
3771 round: Some(round),
3772 };
3773 let (resumed_seat, resumed_out) = run_one(
3774 retry,
3775 Arc::clone(&self.sem),
3776 &ctx,
3777 &mut self.state,
3778 attempts,
3779 )
3780 .await;
3781 seat = resumed_seat;
3782 match resumed_out {
3783 AgentOutcome::Ok(o) => {
3784 cumulative_wait_ms += o.duration_ms;
3785 match verdict::extract_json::<FixReport>(&o.text) {
3786 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3787 self.state.event(
3788 "fix",
3789 format!(
3790 "round {round}: fixer's adoption report recovered after \
3791 {attempts} continuation(s)"
3792 ),
3793 );
3794 return (
3795 seat,
3796 Some(report),
3797 None,
3798 ContinuationRecord {
3799 attempts,
3800 cumulative_wait_ms,
3801 outcome: ContinuationOutcome::Resumed,
3802 },
3803 );
3804 }
3805 Ok(_) => {
3813 last_err = "the reply parsed, but it reported a command whose own CLI \
3814 never confirmed an exit status"
3815 .to_owned();
3816 }
3817 Err(e) => last_err = e.to_string(),
3818 }
3819 }
3820 AgentOutcome::Quota(o) => {
3821 cumulative_wait_ms += o.duration_ms;
3822 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3823 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3824 self.state.quota.push(loss);
3825 self.state.event(
3826 "fix",
3827 format!(
3828 "round {round}: continuation rate limited (quota); not retrying now"
3829 ),
3830 );
3831 return (
3832 seat,
3833 None,
3834 Some("rate limited (quota) while recovering the fix report".to_owned()),
3835 ContinuationRecord {
3836 attempts,
3837 cumulative_wait_ms,
3838 outcome: ContinuationOutcome::QuotaLost,
3839 },
3840 );
3841 }
3842 AgentOutcome::Dropped(o) => {
3843 cumulative_wait_ms += o.duration_ms;
3844 let why = o
3845 .dropped
3846 .as_ref()
3847 .map(|d| d.why.as_str())
3848 .unwrap_or("the CLI ended the stream without delivering its answer");
3849 last_err = format!("the CLI dropped the stream ({why})");
3850 }
3851 AgentOutcome::Failed(e) => last_err = e,
3852 AgentOutcome::ResourceBlocked(e) => {
3855 return (
3856 seat,
3857 None,
3858 Some(format!("the continuation was not started: {e}")),
3859 ContinuationRecord {
3860 attempts: attempts.saturating_sub(1),
3861 cumulative_wait_ms,
3862 outcome: ContinuationOutcome::NoSession,
3863 },
3864 );
3865 }
3866 }
3867 }
3868 }
3869
3870 fn after_implement(&mut self) -> Result<()> {
3871 if !self.state.candidate_preservation.is_empty() {
3872 bail!("candidate preservation incomplete; judging is unsafe");
3873 }
3874 if self.state.leaks.is_empty() {
3876 let cfg = self.state.config.blind.clone();
3877 let mut leaks = Vec::new();
3878 for c in &self.state.candidates {
3879 let Some(patch) =
3880 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3881 else {
3882 continue;
3883 };
3884 leaks.extend(blind::scan(
3885 &format!("candidate {} patch", c.label),
3886 &patch,
3887 &cfg.vendor_tokens,
3888 ));
3889 }
3890 if !leaks.is_empty() {
3891 let summary = leaks
3892 .iter()
3893 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3894 .collect::<Vec<_>>()
3895 .join(", ");
3896 match cfg.on_leak {
3897 LeakPolicy::Fail => {
3898 self.state.status = RunStatus::Failed;
3899 self.state
3900 .event("blind", format!("vendor text in a patch: {summary}"));
3901 self.state.leaks = leaks;
3902 self.state.save()?;
3903 self.settle_questions();
3904 bail!(
3905 "blind.on_leak = \"fail\" and vendor text reached a \
3906 judged patch: {summary}"
3907 );
3908 }
3909 LeakPolicy::Redact => self.state.event(
3910 "blind",
3911 format!("redacting vendor text for judging: {summary}"),
3912 ),
3913 LeakPolicy::Warn => self.state.event(
3914 "blind",
3915 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3916 ),
3917 }
3918 self.state.leaks = leaks;
3919 }
3920 }
3921
3922 if self.state.viable().is_empty() {
3923 if self.state.all_candidates_verified_noop() {
3924 self.state.status = RunStatus::VerifiedNoop;
3935 self.state.save()?;
3936 self.settle_questions();
3937 return Ok(());
3938 }
3939 self.state.status = RunStatus::Failed;
3940 self.state.save()?;
3941 self.settle_questions();
3942 bail!("no candidate produced a change; nothing to judge");
3943 }
3944 self.state.status = RunStatus::Judging;
3945 self.state.save()?;
3946 Ok(())
3947 }
3948
3949 async fn judge(&mut self) -> Result<()> {
3952 let run_id = self.state.id.clone();
3957 let prompts = self.state.config.prompts.clone();
3958 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3959 return Ok(());
3960 }
3961 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3962 if viable.len() == 1 {
3963 self.state.judge_skipped = true;
3970 self.state.event(
3971 "judge",
3972 format!(
3973 "only candidate {} produced a change; judging skipped",
3974 viable[0].label
3975 ),
3976 );
3977 self.state.save()?;
3978 return Ok(());
3979 }
3980 self.state.status = RunStatus::Judging;
3981
3982 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983 let language = self.state.config.graph.language.clone();
3984 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985 let sessions = self.state.config.graph.sessions;
3986 let artifacts = agent::artifacts_dir(&self.state.dir());
3987 let root = self.state.worktree_root();
3988 let base_short = short(&self.state.base_commit);
3989
3990 let mut jobs = Vec::new();
3991 let mut orders = Vec::new();
3992 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3993 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3994 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3995 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3996 let seat_key = format!("judge-{}", j + 1);
3997 let seat = self.seat(&seat_key, &spec.id);
3998 jobs.push(SeatJob {
3999 prompt: prompt::judge(
4000 &self.state.instruction,
4001 &views,
4002 self.roles.judges.len(),
4003 &base_short,
4004 &language,
4005 ),
4006 spec,
4007 seat,
4008 cwd: root.join(format!("judge-{}", j + 1)),
4009 timeout,
4010 allow_write: false,
4011 sessions,
4012 artifacts: artifacts.clone(),
4013 stem: format!("judge-{}", j + 1),
4014 handover: None,
4015 });
4016 }
4017
4018 self.state.event(
4019 "judge",
4020 format!(
4021 "{} judges ranking {} candidates blind",
4022 jobs.len(),
4023 viable.len()
4024 ),
4025 );
4026 let labels_for_check = labels.clone();
4027 let mut quota_losses = Vec::new();
4028 let cache = self.state.config.cache_dir();
4029 let ctx = WaveCtx {
4030 carry_seats: false,
4031 pauses: Some((&self.pause, &self.interrupt)),
4032 run: &run_id,
4033 node: "judge",
4034 prompts: &prompts,
4035 cache: cache.as_deref(),
4036 round: None,
4037 };
4038 let results = ask_json_wave::<Ranking>(
4039 jobs,
4040 Arc::clone(&self.sem),
4041 self.state.config.graph.retries,
4042 &self.roles.judge_roster,
4043 &ctx,
4044 &mut quota_losses,
4045 &mut self.state,
4046 &move |r: &Ranking| r.validate(&labels_for_check),
4047 )
4048 .await;
4049 if self.state.resource_block.is_some() {
4050 return self.stop_for_resource();
4051 }
4052 self.state.quota.extend(quota_losses);
4053
4054 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4055 let agent_id = seat.agent.clone();
4056 self.state.seats.insert(seat.key.clone(), seat);
4057 let mut record = Judgement {
4058 judge: j + 1,
4059 seat: format!("judge-{}", j + 1),
4060 agent: agent_id,
4061 ranking: Vec::new(),
4062 reasons: BTreeMap::new(),
4063 confidence: None,
4064 strengths: BTreeMap::new(),
4065 order: orders[j].clone(),
4066 failed: None,
4067 duration_ms: 0,
4068 };
4069 match res {
4070 Ok((ranking, out)) => {
4071 record.ranking = ranking.normalized();
4072 record.reasons = ranking.reasons;
4073 record.strengths = ranking.strengths;
4074 record.confidence = ranking.confidence;
4075 record.duration_ms = out.duration_ms;
4076 self.state.event(
4077 "judge",
4078 format!(
4079 "judge {} ranked {}",
4080 j + 1,
4081 record.ranking.iter().collect::<String>()
4082 ),
4083 );
4084 }
4085 Err(e) => {
4086 record.failed = Some(e.to_string());
4087 self.state
4088 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4089 }
4090 }
4091 self.state.judgements.push(record);
4092 self.state.save()?;
4093 }
4094 Ok(())
4095 }
4096
4097 async fn deliberate(&mut self) -> Result<()> {
4100 if self.hold_base_sync().await? {
4101 return Ok(());
4102 }
4103 let run_id = self.state.id.clone();
4108 let prompts = self.state.config.prompts.clone();
4109 let rounds = self.state.config.graph.deliberate_rounds;
4110 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4112 return Ok(());
4113 }
4114 let tops: Vec<char> = self
4115 .state
4116 .judgements
4117 .iter()
4118 .filter_map(|j| j.ranking.first().copied())
4119 .collect();
4120 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4121 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4122 self.state.event(
4123 "deliberate",
4124 format!("judges agreed on {} outright; no deliberation", tops[0]),
4125 );
4126 }
4127 self.state.status = RunStatus::Voting;
4128 self.state.save()?;
4129 return Ok(());
4130 }
4131
4132 self.state.status = RunStatus::Deliberating;
4133 self.state.event(
4134 "deliberate",
4135 format!(
4136 "split: first choices were {} — opening {rounds} round(s)",
4137 tops.iter().collect::<String>()
4138 ),
4139 );
4140
4141 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4142 let language = self.state.config.graph.language.clone();
4143 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4144 let sessions = self.state.config.graph.sessions;
4145 let artifacts = agent::artifacts_dir(&self.state.dir());
4146 let root = self.state.worktree_root();
4147 let base_short = short(&self.state.base_commit);
4148
4149 for round in self.state.deliberation.len() + 1..=rounds {
4153 let mut turns: Vec<DeliberationTurn> = Vec::new();
4154 let mut round_losses = Vec::new();
4155 let mut asked = 0;
4156 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4157 if self.state.judgements[j].failed.is_some() {
4158 continue;
4159 }
4160 let seat_key = format!("judge-{}", j + 1);
4161 let spec = self.occupant(&seat_key, spec);
4162 let mut seat = self.seat(&seat_key, &spec.id);
4163 let transcript = self.transcript(&turns, j);
4164 let build = |context: Option<&str>| {
4165 prompt::deliberate(
4166 &self.state.instruction,
4167 context,
4168 &transcript,
4169 round,
4170 rounds,
4171 &language,
4172 )
4173 };
4174 let block = self.candidate_block(&viable, &base_short);
4175 let full = build(Some(&block));
4176 let text = if has_context(&spec, &seat, sessions) {
4177 build(None)
4178 } else {
4179 full.clone()
4180 };
4181 let job = SeatJob {
4182 spec,
4183 seat: seat.clone(),
4184 prompt: text,
4185 cwd: root.join(format!("judge-{}", j + 1)),
4186 timeout,
4187 allow_write: false,
4188 sessions,
4189 artifacts: artifacts.clone(),
4190 stem: format!("delib-{round}-judge-{}", j + 1),
4191 handover: Some(full),
4192 };
4193 let cache = self.state.config.cache_dir();
4194 let ctx = WaveCtx {
4195 carry_seats: false,
4196 pauses: Some((&self.pause, &self.interrupt)),
4197 run: &run_id,
4198 node: "deliberate",
4199 prompts: &prompts,
4200 cache: cache.as_deref(),
4201 round: None,
4202 };
4203 let mut losses = Vec::new();
4207 asked += 1;
4208 let mut results = ask_wave_with::<String>(
4209 vec![job],
4210 Arc::clone(&self.sem),
4211 0,
4212 &self.roles.judge_roster,
4213 &ctx,
4214 &mut losses,
4215 &mut self.state,
4216 &|text: &str| {
4217 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4218 },
4219 )
4220 .await;
4221 if self.state.resource_block.is_some() {
4222 return self.stop_for_resource();
4223 }
4224 round_losses.extend(losses.iter().cloned());
4225 self.state.quota.extend(losses);
4226 let (updated, res, _) = results.pop().expect("one job in, one result out");
4227 seat = updated;
4228 let agent_id = seat.agent.clone();
4229 self.state.seats.insert(seat.key.clone(), seat);
4230 let body = match res {
4231 Ok((body, _)) => body,
4232 Err(e) => {
4235 self.state
4236 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4237 continue;
4238 }
4239 };
4240 let tentative = verdict::extract_json::<Position>(&body)
4241 .ok()
4242 .and_then(|p| p.tentative)
4243 .and_then(|s| s.trim().chars().next())
4244 .map(|c| c.to_ascii_uppercase());
4245 self.state.event(
4246 "deliberate",
4247 format!(
4248 "round {round}: judge {} now favours {}",
4249 j + 1,
4250 tentative.map_or("—".to_owned(), |c| c.to_string())
4251 ),
4252 );
4253 turns.push(DeliberationTurn {
4254 judge: j + 1,
4255 agent: agent_id,
4256 body: blind::sanitize_prose(&body, &self.state.config.blind),
4257 tentative,
4258 });
4259 }
4260 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4261 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4262 return self.stop_for_resource();
4263 }
4264 self.state
4265 .deliberation
4266 .push(DeliberationRound { round, turns });
4267 self.state.save()?;
4268 }
4269
4270 self.state.status = RunStatus::Voting;
4271 self.state.save()?;
4272 Ok(())
4273 }
4274
4275 async fn vote(&mut self) -> Result<()> {
4278 if self.hold_base_sync().await? {
4279 return Ok(());
4280 }
4281 let run_id = self.state.id.clone();
4286 let prompts = self.state.config.prompts.clone();
4287 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4288 return Ok(());
4289 }
4290 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4291 if viable.len() == 1 {
4292 return Ok(());
4293 }
4294 self.state.status = RunStatus::Voting;
4295
4296 let language = self.state.config.graph.language.clone();
4297 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4298 let sessions = self.state.config.graph.sessions;
4299 let artifacts = agent::artifacts_dir(&self.state.dir());
4300 let root = self.state.worktree_root();
4301 let base_short = short(&self.state.base_commit);
4302 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4303
4304 let mut jobs = Vec::new();
4305 let mut seats_at = Vec::new();
4306 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4307 if self
4308 .state
4309 .judgements
4310 .get(j)
4311 .is_some_and(|r| r.failed.is_some())
4312 {
4313 continue;
4314 }
4315 let seat_key = format!("judge-{}", j + 1);
4316 let spec = self.occupant(&seat_key, spec);
4317 let seat = self.seat(&seat_key, &spec.id);
4318 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4319 let text = if has_context(&spec, &seat, sessions) {
4320 prompt::final_vote(&viable, &language)
4321 } else {
4322 full.clone()
4323 };
4324 jobs.push(SeatJob {
4325 spec,
4326 seat,
4327 prompt: text,
4328 cwd: root.join(format!("judge-{}", j + 1)),
4329 timeout,
4330 allow_write: false,
4331 sessions,
4332 artifacts: artifacts.clone(),
4333 stem: format!("vote-judge-{}", j + 1),
4334 handover: Some(full),
4335 });
4336 seats_at.push(j);
4337 }
4338
4339 self.state.event(
4340 "vote",
4341 format!(
4342 "collecting {} final votes one by one, privately",
4343 jobs.len()
4344 ),
4345 );
4346 let allowed = viable.clone();
4347 let mut quota_losses = Vec::new();
4348 let cache = self.state.config.cache_dir();
4349 let ctx = WaveCtx {
4350 carry_seats: false,
4351 pauses: Some((&self.pause, &self.interrupt)),
4352 run: &run_id,
4353 node: "vote",
4354 prompts: &prompts,
4355 cache: cache.as_deref(),
4356 round: None,
4357 };
4358 let results = ask_json_wave::<FinalVote>(
4359 jobs,
4360 Arc::clone(&self.sem),
4361 self.state.config.graph.retries,
4362 &self.roles.judge_roster,
4363 &ctx,
4364 &mut quota_losses,
4365 &mut self.state,
4366 &move |v: &FinalVote| match v.label() {
4367 Some(c) if allowed.contains(&c) => Ok(()),
4368 other => bail!("vote {other:?} is not one of {allowed:?}"),
4369 },
4370 )
4371 .await;
4372 if self.state.resource_block.is_some() {
4373 return self.stop_for_resource();
4374 }
4375 self.state.quota.extend(quota_losses);
4376
4377 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4378 let agent_id = seat.agent.clone();
4379 self.state.seats.insert(seat.key.clone(), seat);
4380 let initial = self
4381 .state
4382 .judgements
4383 .get(j)
4384 .and_then(|r| r.ranking.first().copied());
4385 let mut record = VoteRecord {
4386 judge: j + 1,
4387 agent: agent_id,
4388 vote: None,
4389 reason: String::new(),
4390 changed: false,
4391 };
4392 match res {
4393 Ok((v, _)) => {
4394 record.vote = v.label();
4395 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4396 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4397 self.state.event(
4398 "vote",
4399 format!(
4400 "judge {} voted {}{}",
4401 j + 1,
4402 record.vote.unwrap_or('?'),
4403 if record.changed { " (changed)" } else { "" }
4404 ),
4405 );
4406 }
4407 Err(e) => {
4408 self.state
4409 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4410 }
4411 }
4412 self.state.votes.push(record);
4413 self.state.save()?;
4414 }
4415 Ok(())
4416 }
4417
4418 fn tally(&mut self) -> Result<()> {
4421 if self.state.tally.is_some() {
4422 return Ok(());
4423 }
4424 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4425 let tops: Vec<char> = self
4426 .state
4427 .judgements
4428 .iter()
4429 .filter_map(|j| j.ranking.first().copied())
4430 .collect();
4431 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4432
4433 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4436 let mut cast: Vec<char> = Vec::new();
4437 for (i, j) in self.state.judgements.iter().enumerate() {
4438 let vote = self
4439 .state
4440 .votes
4441 .iter()
4442 .find(|v| v.judge == i + 1)
4443 .and_then(|v| v.vote)
4444 .or_else(|| j.ranking.first().copied());
4445 if let Some(v) = vote {
4446 *first_choice.entry(v).or_insert(0) += 1;
4447 cast.push(v);
4448 }
4449 }
4450
4451 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4452 for j in &self.state.judgements {
4453 let n = j.ranking.len();
4454 for (pos, label) in j.ranking.iter().enumerate() {
4455 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4456 }
4457 }
4458
4459 let best = first_choice.values().copied().max().unwrap_or(0);
4460 let mut leaders: Vec<char> = first_choice
4461 .iter()
4462 .filter(|(_, v)| **v == best)
4463 .map(|(k, _)| *k)
4464 .collect();
4465 let mut tie_break = None;
4466 if leaders.len() > 1 {
4467 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4468 let borda_leaders: Vec<char> = leaders
4469 .iter()
4470 .copied()
4471 .filter(|l| borda[l] == top_borda)
4472 .collect();
4473 tie_break = Some(if borda_leaders.len() == 1 {
4474 format!(
4475 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4476 leaders.len()
4477 )
4478 } else {
4479 format!(
4480 "{} way tie on both first-choice votes and Borda points, broken by label order",
4481 leaders.len()
4482 )
4483 });
4484 leaders = borda_leaders;
4485 leaders.sort_unstable();
4486 }
4487 let winner = *leaders
4488 .first()
4489 .or(viable.first())
4490 .context("no candidate to declare a winner from")?;
4491
4492 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4493 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4494 let deliberated = !self.state.deliberation.is_empty();
4495
4496 let quota_seats: std::collections::BTreeSet<&str> =
4500 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4501 let mut present = 0usize;
4502 for (i, j) in self.state.judgements.iter().enumerate() {
4503 if quota_seats.contains(j.seat.as_str()) {
4504 continue;
4505 }
4506 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4507 let voted = self
4508 .state
4509 .votes
4510 .iter()
4511 .any(|v| v.judge == i + 1 && v.vote.is_some());
4512 if ranked || voted {
4513 present += 1;
4514 }
4515 }
4516 let needs_quorum = viable.len() > 1;
4522 let judges_total = if needs_quorum {
4523 self.roles.judges.len()
4524 } else {
4525 0
4526 };
4527 let quorum = if needs_quorum {
4528 judges_total / 2 + 1
4529 } else {
4530 0
4531 };
4532 let met_quorum = !needs_quorum || present >= quorum;
4533 let uncontested = (!needs_quorum).then(|| {
4534 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4535 });
4536
4537 self.state.event(
4538 "tally",
4539 match &uncontested {
4540 Some(reason) => format!("winner {winner} — {reason}"),
4541 None => format!(
4542 "winner {winner} — votes {} | initial {} | {} changed | \
4543 {present}/{judges_total} judges{}",
4544 first_choice
4545 .iter()
4546 .map(|(k, v)| format!("{k}:{v}"))
4547 .collect::<Vec<_>>()
4548 .join(" "),
4549 if unanimous_initial {
4550 "unanimous"
4551 } else {
4552 "split"
4553 },
4554 changed_votes,
4555 if met_quorum {
4556 String::new()
4557 } else {
4558 format!(" — below quorum ({quorum} required)")
4559 },
4560 ),
4561 },
4562 );
4563 if !met_quorum {
4564 self.state.event(
4565 "stall",
4566 format!(
4567 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4568 the run stops here, resumable"
4569 ),
4570 );
4571 }
4572 self.state.tally = Some(Tally {
4573 first_choice,
4574 borda,
4575 winner,
4576 rankings: tops.len(),
4577 unanimous_initial,
4578 deliberated,
4579 changed_votes,
4580 unanimous_final,
4581 tie_break,
4582 judges: judges_total,
4583 present,
4584 quorum,
4585 met_quorum,
4586 uncontested,
4587 });
4588 self.state.status = if met_quorum {
4589 RunStatus::Reviewing
4590 } else {
4591 RunStatus::Stalled
4592 };
4593 self.state.save()?;
4594 Ok(())
4595 }
4596
4597 #[allow(clippy::too_many_lines)]
4618 async fn recover_stall(&mut self) -> Result<bool> {
4619 let run_id = self.state.id.clone();
4624 let prompts = self.state.config.prompts.clone();
4625 let quota_seats: BTreeSet<&str> =
4630 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4631 let absent: Vec<String> = self
4632 .state
4633 .judgements
4634 .iter()
4635 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4636 .map(|j| j.seat.clone())
4637 .collect();
4638 if absent.is_empty() {
4639 return Ok(false);
4640 }
4641 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4642 if viable.len() <= 1 {
4643 return Ok(false);
4644 }
4645 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4646 let language = self.state.config.graph.language.clone();
4647 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4648 let sessions = self.state.config.graph.sessions;
4649 let artifacts = agent::artifacts_dir(&self.state.dir());
4650 let root = self.state.worktree_root();
4651 let base_short = short(&self.state.base_commit);
4652 let candidates: Vec<Candidate> = viable.clone();
4653
4654 let mut positions: Vec<usize> = absent
4656 .iter()
4657 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4658 .collect();
4659 if positions.is_empty() {
4660 return Ok(false);
4661 }
4662 positions.sort_unstable();
4663 positions.dedup();
4664
4665 let mut judge_jobs = Vec::new();
4667 for &j in &positions {
4668 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4669 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4670 let seat_key = format!("judge-{}", j + 1);
4671 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4672 let seat = self.seat(&seat_key, &spec.id);
4673 judge_jobs.push(SeatJob {
4674 spec,
4675 seat,
4676 prompt: prompt::judge(
4677 &self.state.instruction,
4678 &views,
4679 self.roles.judges.len(),
4680 &base_short,
4681 &language,
4682 ),
4683 cwd: root.join(seat_key),
4684 timeout,
4685 allow_write: false,
4686 sessions,
4687 artifacts: artifacts.clone(),
4688 stem: format!("judge-{}-recover", j + 1),
4689 handover: None,
4690 });
4691 }
4692
4693 let labels_for_check = labels.clone();
4694 let mut judge_losses = Vec::new();
4695 let retries = self.state.config.graph.retries;
4696 let cache = self.state.config.cache_dir();
4697 let ctx = WaveCtx {
4698 carry_seats: false,
4699 pauses: Some((&self.pause, &self.interrupt)),
4700 run: &run_id,
4701 node: "judge",
4702 prompts: &prompts,
4703 cache: cache.as_deref(),
4704 round: None,
4705 };
4706 let results = ask_json_wave::<Ranking>(
4707 judge_jobs,
4708 Arc::clone(&self.sem),
4709 retries,
4710 &self.roles.judge_roster,
4711 &ctx,
4712 &mut judge_losses,
4713 &mut self.state,
4714 &move |r: &Ranking| r.validate(&labels_for_check),
4715 )
4716 .await;
4717 if self.state.resource_block.is_some() {
4718 self.stop_for_resource()?;
4719 return Ok(false);
4720 }
4721
4722 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4724 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4725 let agent_id = seat.agent.clone();
4726 self.state.seats.insert(seat.key.clone(), seat);
4727 let record = &mut self.state.judgements[j];
4728 match res {
4729 Ok((ranking, out)) => {
4730 record.agent = agent_id;
4731 record.ranking = ranking.normalized();
4732 record.reasons = ranking.reasons;
4733 record.strengths = ranking.strengths;
4734 record.confidence = ranking.confidence;
4735 record.failed = None;
4736 record.duration_ms = out.duration_ms;
4737 recovered.insert(j);
4738 self.state.event(
4739 "recover",
4740 format!("judge {} ranked again after the limit", j + 1),
4741 );
4742 }
4743 Err(e) => {
4744 self.state
4745 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4746 }
4747 }
4748 }
4749
4750 let mut vote_jobs = Vec::new();
4752 let mut vote_pos: Vec<usize> = Vec::new();
4753 for &j in &recovered {
4754 let seat_key = format!("judge-{}", j + 1);
4755 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4756 let seat = self.seat(&seat_key, &spec.id);
4757 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4758 let text = if has_context(&spec, &seat, sessions) {
4759 prompt::final_vote(&labels, &language)
4760 } else {
4761 full.clone()
4762 };
4763 vote_jobs.push(SeatJob {
4764 spec,
4765 seat,
4766 prompt: text,
4767 cwd: root.join(seat_key),
4768 timeout,
4769 allow_write: false,
4770 sessions,
4771 artifacts: artifacts.clone(),
4772 stem: format!("vote-judge-{}-recover", j + 1),
4773 handover: Some(full),
4774 });
4775 vote_pos.push(j);
4776 }
4777 let allowed = labels.clone();
4778 let mut vote_losses = Vec::new();
4779 let vote_retries = self.state.config.graph.retries;
4780 let vote_cache = self.state.config.cache_dir();
4781 let ctx = WaveCtx {
4782 carry_seats: false,
4783 pauses: Some((&self.pause, &self.interrupt)),
4784 run: &run_id,
4785 node: "vote",
4786 prompts: &prompts,
4787 cache: vote_cache.as_deref(),
4788 round: None,
4789 };
4790 let votes = ask_json_wave::<FinalVote>(
4791 vote_jobs,
4792 Arc::clone(&self.sem),
4793 vote_retries,
4794 &self.roles.judge_roster,
4795 &ctx,
4796 &mut vote_losses,
4797 &mut self.state,
4798 &move |v: &FinalVote| match v.label() {
4799 Some(c) if allowed.contains(&c) => Ok(()),
4800 other => bail!("vote {other:?} is not one of {allowed:?}"),
4801 },
4802 )
4803 .await;
4804 if self.state.resource_block.is_some() {
4805 self.stop_for_resource()?;
4806 return Ok(false);
4807 }
4808 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4809 let agent_id = seat.agent.clone();
4810 self.state.seats.insert(seat.key.clone(), seat);
4811 match res {
4812 Ok((v, _)) => {
4813 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4814 rec.vote = v.label();
4815 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4816 } else {
4817 self.state.votes.push(VoteRecord {
4818 judge: j + 1,
4819 agent: agent_id,
4820 vote: v.label(),
4821 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4822 changed: false,
4823 });
4824 }
4825 self.state.event(
4826 "recover",
4827 format!("judge {} voted again after the limit", j + 1),
4828 );
4829 }
4830 Err(e) => {
4831 self.state
4832 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4833 }
4834 }
4835 }
4836
4837 let recovered_keys: BTreeSet<String> = recovered
4841 .iter()
4842 .map(|&j| format!("judge-{}", j + 1))
4843 .collect();
4844 self.state
4845 .quota
4846 .retain(|q| !recovered_keys.contains(&q.seat));
4847 for loss in judge_losses.into_iter().chain(vote_losses) {
4851 if recovered_keys.contains(&loss.seat) {
4852 continue;
4853 }
4854 self.state.quota.retain(|q| q.seat != loss.seat);
4855 self.state.quota.push(loss);
4856 }
4857
4858 self.state.tally = None;
4860 self.tally()?;
4861 Ok(self
4862 .state
4863 .tally
4864 .as_ref()
4865 .map(|t| t.met_quorum)
4866 .unwrap_or(false))
4867 }
4868
4869 async fn graft(&mut self) -> Result<()> {
4885 let Err(e) = self.graft_inner().await else {
4886 return Ok(());
4887 };
4888 if e.is::<crate::repo_lock::ResourceBlocked>() {
4889 return Err(e);
4890 }
4891 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4892 self.state.status = RunStatus::Blocked;
4893 self.state.event("graft", format!("blocked: {e:#}"));
4894 self.state.save()?;
4895 return Err(e);
4896 }
4897 if let Some(rec) = self.state.graft.clone()
4902 && !rec.outcome.settled()
4903 {
4904 if let Some(winner) = self.state.winner().cloned() {
4905 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4906 if re.is::<crate::repo_lock::ResourceBlocked>() {
4907 return Err(re);
4908 }
4909 self.state.status = RunStatus::Blocked;
4910 self.state.event("graft", format!("blocked: {re:#}"));
4911 self.state.save()?;
4912 return Err(re);
4913 }
4914 }
4915 if let Some(g) = self.state.graft.as_mut() {
4916 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4917 }
4918 }
4919 self.state.event("graft", format!("skipped: {e:#}"));
4920 if let Err(e) = self.state.save() {
4921 tracing::warn!("save after graft: {e:#}");
4922 }
4923 Ok(())
4924 }
4925
4926 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4929 let _coordination = self.coordinate(wt).await?;
4930 crate::recovery::restore_repair(wt, pre)
4931 .await
4932 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4933 }
4934
4935 fn graft_skipped(&mut self, why: &str) {
4937 let message = format!("skipped: {why}");
4938 if !self
4939 .state
4940 .events
4941 .iter()
4942 .any(|e| e.node == "graft" && e.message == message)
4943 {
4944 self.state.event("graft", message);
4945 }
4946 }
4947
4948 async fn graft_inner(&mut self) -> Result<()> {
4949 if let Some(rec) = self.state.graft.clone()
4952 && !rec.outcome.settled()
4953 {
4954 if self.state.reviews.is_empty()
4958 && let Some(winner) = self.state.winner().cloned()
4959 {
4960 self.graft_restore(&winner.worktree, &rec.pre).await?;
4961 }
4962 if rec.outcome == GraftOutcome::WaitingForRestore {
4963 self.state.graft = None;
4964 self.state.save()?;
4965 } else {
4966 if let Some(g) = self.state.graft.as_mut() {
4967 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4968 }
4969 self.state.event(
4970 "graft",
4971 "rolled back: the previous round was interrupted; the winner is untouched",
4972 );
4973 self.state.save()?;
4974 return Ok(());
4975 }
4976 }
4977 let plan = match graft::plan(&self.state) {
4978 Ok(plan) => plan,
4979 Err(skip) => {
4980 if skip != graft::Skip::AlreadyDone {
4981 self.graft_skipped(skip.reason());
4982 self.state.save()?;
4983 }
4984 return Ok(());
4985 }
4986 };
4987 let Some(winner) = self.state.winner().cloned() else {
4988 return Ok(());
4989 };
4990 let wt = winner.worktree.clone();
4991 if !git::is_clean(&wt).await? {
4992 self.graft_skipped("the winner's worktree has uncommitted changes");
4993 self.state.save()?;
4994 return Ok(());
4995 }
4996 let pre = git::rev_parse(&wt, "HEAD").await?;
4997 self.state.graft = Some(GraftRecord {
5000 pre: pre.clone(),
5001 post: None,
5002 lines: None,
5003 limit: None,
5004 agent: None,
5005 outcome: GraftOutcome::Running,
5006 requested: graft::requested(&plan),
5007 adoption: None,
5008 adoption_error: None,
5009 });
5010 self.state.event(
5011 "graft",
5012 format!(
5013 "porting strengths of candidate(s) {} onto {}",
5014 plan.strengths
5015 .iter()
5016 .map(|(l, _, _)| l.to_string())
5017 .collect::<Vec<_>>()
5018 .join(", "),
5019 plan.winner
5020 ),
5021 );
5022 self.state.save()?;
5023
5024 let fix_prompt = prompt::graft(
5025 &self.state.instruction,
5026 plan.winner,
5027 &plan.strengths,
5028 &self.state.config.graph.language,
5029 );
5030 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5031 let sessions = self.state.config.graph.sessions;
5032 let artifacts = agent::artifacts_dir(&self.state.dir());
5033 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5034 let (_, seat, out) = self
5035 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5036 SeatJob {
5037 prompt: fix_prompt.clone(),
5038 spec,
5039 seat,
5040 cwd: wt.clone(),
5041 timeout,
5042 allow_write: true,
5043 sessions,
5044 artifacts: artifacts.clone(),
5045 stem: "graft".to_owned(),
5046 handover: None,
5047 }
5048 })
5049 .await;
5050 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5051 self.state.graft.as_mut().expect("graft record").outcome =
5054 GraftOutcome::WaitingForRestore;
5055 return self.stop_for_resource();
5056 }
5057 let mut why: Option<String> = None;
5058 match &out {
5059 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5060 why = Some("the fixer produced nothing usable".to_owned());
5061 }
5062 AgentOutcome::Ok(_) => {}
5063 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5064 AgentOutcome::Quota(o) => {
5065 self.state.quota.push(QuotaLoss::from_report(
5066 seat.key.clone(),
5067 "graft",
5068 o.quota.as_ref(),
5069 ));
5070 why = Some("rate limited (quota)".to_owned());
5071 }
5072 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5073 why = Some(format!(
5074 "the fixer failed: {}",
5075 e.lines()
5076 .next()
5077 .unwrap_or("")
5078 .chars()
5079 .take(160)
5080 .collect::<String>()
5081 ));
5082 }
5083 }
5084 let agent_id = seat.agent.clone();
5085 self.state.seats.insert(seat.key.clone(), seat);
5086
5087 let mut post = None;
5088 let mut lines = None;
5089 let _coordination = self.coordinate(&wt).await?;
5090 if why.is_none() {
5091 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5092 if let Ok(r) = &rescued {
5093 self.state.note_withheld("graft", &r.withheld);
5094 }
5095 let head = git::rev_parse(&wt, "HEAD").await?;
5096 let dirty = match &rescued {
5100 Ok(r) => git::status_porcelain(&wt)
5101 .await
5102 .map(|st| {
5103 st.lines().any(|l| {
5104 !r.withheld
5105 .iter()
5106 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5107 })
5108 })
5109 .unwrap_or(true),
5110 Err(_) => true,
5111 };
5112 if dirty {
5113 why = Some("the fixer left changes that could not be committed".to_owned());
5114 } else if head == pre {
5115 why = Some("the fixer changed nothing".to_owned());
5116 } else {
5117 lines = git::changed_lines(&wt, &pre, &head).await;
5118 post = Some(head);
5119 }
5120 }
5121 let (adoption, adoption_error) = match &out {
5122 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5123 _ => (None, None),
5124 };
5125 let requested = graft::requested(&plan);
5126 let outcome = match why {
5127 None => {
5128 self.state.event(
5129 "graft",
5130 format!(
5131 "applied: {}",
5132 lines.map_or_else(
5133 || "changed lines unavailable".to_owned(),
5134 |n| format!("{n} changed lines")
5135 )
5136 ),
5137 );
5138 GraftOutcome::Applied
5139 }
5140 Some(why) => {
5141 crate::recovery::restore_repair(&wt, &pre)
5144 .await
5145 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5146 self.state.event(
5147 "graft",
5148 format!("rolled back: {why}; the winner is untouched"),
5149 );
5150 GraftOutcome::RolledBack(why)
5151 }
5152 };
5153 self.state.graft = Some(GraftRecord {
5154 pre,
5155 post,
5156 lines,
5157 limit: None,
5158 agent: Some(agent_id),
5159 outcome,
5160 requested,
5161 adoption,
5162 adoption_error,
5163 });
5164 self.state.save()?;
5165 Ok(())
5166 }
5167
5168 async fn fold_losers(&mut self) -> Result<()> {
5171 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5172 return Ok(());
5173 };
5174 if !self.state.candidate_preservation.is_empty() {
5175 bail!("candidate preservation incomplete; refusing cleanup");
5176 }
5177 let repo = self.state.repo.clone();
5178 let _coordination = self.coordinate(&repo).await?;
5179 let mut folded = Vec::new();
5180 for i in 0..self.state.candidates.len() {
5181 let c = &self.state.candidates[i];
5182 if c.label == winner || c.folded {
5183 continue;
5184 }
5185 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5186 remove_preserved_worktree(&repo, &wt).await?;
5187 if git::branch_exists(&repo, &branch).await?
5188 && !git::branch_delete(&repo, &branch).await?
5189 {
5190 bail!("candidate branch removal refused");
5191 }
5192 self.state.candidates[i].folded = true;
5193 self.state.save()?;
5194 folded.push(label.to_string());
5195 }
5196 let root = self.state.worktree_root();
5198 for j in 1..=self.roles.judges.len() {
5199 let wt = root.join(format!("judge-{j}"));
5200 if wt.exists() {
5201 remove_preserved_worktree(&repo, &wt).await?;
5202 }
5203 }
5204 if self.state.config.graph.advise {
5207 for k in 1..=self.state.config.graph.advisors {
5208 let wt = root.join(format!("advisor-{k}"));
5209 if wt.exists() {
5210 remove_preserved_worktree(&repo, &wt).await?;
5211 }
5212 }
5213 }
5214 if !folded.is_empty() {
5215 self.state
5216 .event("fold", format!("folded candidates {}", folded.join(", ")));
5217 self.state.save()?;
5218 }
5219 Ok(())
5220 }
5221
5222 async fn hold_base_sync(&mut self) -> Result<bool> {
5225 self.hold_base_sync_as(None).await
5226 }
5227
5228 async fn hold_base_sync_kind(
5229 &mut self,
5230 kind: crate::base_sync_recovery::StopKind,
5231 ) -> Result<()> {
5232 self.hold_base_sync_as(Some(kind)).await?;
5233 Ok(())
5234 }
5235
5236 async fn hold_base_sync_as(
5237 &mut self,
5238 kind: Option<crate::base_sync_recovery::StopKind>,
5239 ) -> Result<bool> {
5240 if !self.state.base_sync_blocked() {
5241 self.state.base_sync_recovery.stop = None;
5242 return Ok(false);
5243 }
5244 let head = match self.state.winner() {
5245 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5246 .await
5247 .unwrap_or_default(),
5248 None => String::new(),
5249 };
5250 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5251 && !authorization.consumed
5252 && authorization.stop.head != head
5253 {
5254 authorization.refuse(format!(
5255 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5256 authorization.stop.head
5257 ));
5258 }
5259 self.state.record_base_sync_stop(&head);
5260 if let Some(kind) = kind {
5261 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5262 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5263 stop.kind = kind;
5264 }
5265 }
5266 self.state.save()?;
5267 Ok(true)
5268 }
5269
5270 async fn sync_to_base(&mut self) -> Result<()> {
5312 if self.state.status == RunStatus::AlreadyInBase {
5313 return Ok(());
5314 }
5315 self.hold_base_sync().await?;
5316 let conflicted = self
5317 .state
5318 .base_sync
5319 .as_ref()
5320 .is_some_and(|s| s.conflict.is_some());
5321 let Some(winner) = self.state.winner().cloned() else {
5322 return Ok(());
5323 };
5324
5325 let repo = self.state.repo.clone();
5326 let remote = self.state.config.merge.remote.clone();
5327 let base_branch = self.state.base_branch.clone();
5328 let tracking = format!("{remote}/{base_branch}");
5329
5330 let fetched_base =
5331 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5332 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5337 return Ok(());
5338 };
5339
5340 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5341 self.state.ensure_base_episode();
5342 let mut pending = self
5343 .state
5344 .base_sync_recovery
5345 .operations
5346 .last()
5347 .filter(|o| !o.finished)
5348 .cloned();
5349 let scratch = self.state.dir().join("base-sync");
5350 if git::rebase_in_progress(&scratch).await {
5351 if pending.is_none()
5354 && self.state.base_sync_recovery.operations.is_empty()
5355 && !conflicted
5356 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5357 .await
5358 .ok()
5359 .as_deref()
5360 == Some(head.as_str())
5361 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5362 && git::is_clean(&winner.worktree).await?
5363 {
5364 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5365 if self.state.base_sync.is_none() {
5366 self.state.base_sync = Some(BaseSync {
5367 tip: saved_tip.clone(),
5368 behind: 0,
5369 attempts: 0,
5370 conflict: None,
5371 already_in: None,
5372 });
5373 }
5374 self.state.reserve_base_operation(&head, &saved_tip);
5375 self.state.event(
5376 "base-recovery",
5377 "imported standing legacy rebase as a spent synchronization reservation",
5378 );
5379 self.state.save()?;
5380 pending = self.state.base_sync_recovery.operations.last().cloned();
5381 }
5382 let expected = pending.as_ref().map(|op| op.tip.as_str());
5383 if expected.is_none()
5384 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5385 {
5386 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5387 if self.state.base_sync.is_none() {
5388 self.state.base_sync = Some(BaseSync {
5389 tip: tip.clone(),
5390 behind: 0,
5391 attempts: 0,
5392 conflict: None,
5393 already_in: None,
5394 });
5395 }
5396 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5397 self.state.event("land", why);
5398 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5399 .await?;
5400 return Ok(());
5401 }
5402 }
5403 if let Some(op) = &pending {
5404 let mut replay_proven =
5406 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5407 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5408 let wait = repository_wait(&self.state, None);
5409 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5410 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5411 Ok(true) => {
5412 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5413 self.state.save()?;
5414 git::worktree_remove(&repo, &scratch).await.ok();
5415 replay_proven = true;
5416 }
5417 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5418 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5419 }
5420 }
5421 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5422 if self.state.foreign_checkout.is_some() {
5423 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5424 .await
5425 .ok()
5426 .as_deref()
5427 == Some(op.head.as_str())
5428 && git::rev_parse(&repo, &format!("{head}^2"))
5429 .await
5430 .ok()
5431 .as_deref()
5432 == Some(op.tip.as_str());
5433 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5434 .await
5435 .is_ok_and(|m| m.is_empty())
5436 && let Ok(added) =
5437 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5438 {
5439 replay_proven = true;
5440 for commit in added {
5441 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5442 replay_proven = false;
5443 break;
5444 }
5445 }
5446 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5447 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5448 }
5449 }
5450 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5454 || (head != op.head
5455 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5456 .await?
5457 .is_empty())
5458 || (!git::is_clean(&winner.worktree).await?
5459 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5460 .await
5461 .is_ok_and(|o| o.ok()))
5462 || !replay_proven
5463 {
5464 let why = format!(
5465 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5466 op.id
5467 );
5468 let sync = self
5469 .state
5470 .base_sync
5471 .as_mut()
5472 .context("reserved synchronization has no lifetime accounting")?;
5473 sync.conflict = Some(why.clone());
5474 self.state.event("land", why);
5475 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5476 .await?;
5477 return Ok(());
5478 }
5479 if head != op.head {
5480 git::sync_to_head(&winner.worktree).await?;
5481 }
5482 tip = op.tip.clone();
5483 }
5484 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5485 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5486 if pending.is_none()
5487 && !conflicted
5488 && fetched_base
5489 && self
5490 .state
5491 .base_sync_recovery
5492 .episodes
5493 .last()
5494 .is_some_and(|e| e.boundary.is_some())
5495 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5496 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5497 && owner.runs.last() == Some(&self.state.id)
5498 && owner.status != crate::queue::TaskStatus::Done
5499 && !owner.fresh_start
5500 && owner.review_branch.is_none()
5501 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5502 && let Some(approval) = owner.resumption.as_ref()
5503 && approval.instruction == owner.instruction
5504 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5505 && git::is_clean(&winner.worktree).await?
5506 && self.state.renew_base_episode(&task, approval, &head, &tip)
5507 {
5508 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5509 self.state.save()?;
5510 }
5511
5512 if (behind > 0 || conflicted || head == tip)
5521 && self
5522 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5523 .await?
5524 {
5525 return Ok(());
5526 }
5527 if conflicted && self.state.base_recovery_pending() {
5528 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5529 let owner =
5530 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5531 let owner_valid = owner.as_ref().is_ok_and(|t| {
5532 t.runs.last() == Some(&self.state.id)
5533 && t.status != crate::queue::TaskStatus::Done
5534 && !t.fresh_start
5535 && t.review_branch.is_none()
5536 });
5537 let invalidation = if !owner_valid {
5538 Some("latest task ownership changed")
5539 } else if !owner.as_ref().is_ok_and(|t| {
5540 authorization
5541 .generation
5542 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5543 }) {
5544 Some("owner approval generation changed")
5545 } else if let Some(scope) = &authorization.scope {
5546 if scope.tip != tip {
5547 Some("fetched base changed")
5548 } else if !self
5549 .state
5550 .base_sync_recovery
5551 .episodes
5552 .last()
5553 .is_some_and(|e| e.id == scope.episode)
5554 {
5555 Some("active synchronization interval changed")
5556 } else if !owner
5557 .as_ref()
5558 .is_ok_and(|t| t.instruction == scope.instruction)
5559 {
5560 Some("task instruction changed")
5561 } else {
5562 None
5563 }
5564 } else {
5565 None
5566 };
5567 if let Some(reason) = invalidation {
5568 let authorization = self
5569 .state
5570 .base_sync_recovery
5571 .authorizations
5572 .last_mut()
5573 .unwrap();
5574 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5575 self.state.save()?;
5576 return Ok(());
5577 }
5578 if !fetched_base
5579 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5580 || !git::is_clean(&winner.worktree).await?
5581 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5582 {
5583 return Ok(());
5584 }
5585 }
5586 let recovery = conflicted && self.state.consume_base_recovery(&head);
5587 if conflicted {
5588 self.state.save()?;
5590 if !recovery {
5591 return Ok(());
5592 }
5593 }
5594
5595 if behind == 0 {
5596 if let Some(from) = self
5605 .state
5606 .rebase_fixes
5607 .iter()
5608 .rev()
5609 .find_map(|r| r.from.clone())
5610 && from != head
5611 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5612 .await
5613 .is_ok_and(|o| o.ok())
5614 {
5615 git::sync_to_head(&winner.worktree).await?;
5616 }
5617 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5623 if let Some(why) = &conflict {
5624 self.state.status = RunStatus::Blocked;
5625 self.state.event("land", why.clone());
5626 }
5627 self.state.base_sync = Some(BaseSync {
5628 tip,
5629 behind: 0,
5630 attempts,
5631 conflict,
5632 already_in: None,
5633 });
5634 if pending.is_some() {
5635 self.state.finish_base_operation(Some(&head));
5636 }
5637 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5638 .await?;
5639 self.state.save()?;
5640 return Ok(());
5641 }
5642
5643 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5644 let why = format!(
5645 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5646 synchronization(s); synchronizing again would only race it",
5647 winner.branch
5648 );
5649 self.state.status = RunStatus::Blocked;
5650 self.state.base_sync = Some(BaseSync {
5651 tip,
5652 behind,
5653 attempts,
5654 conflict: Some(why.clone()),
5655 already_in: None,
5656 });
5657 self.state.event("land", why);
5658 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5659 .await?;
5660 self.state.save()?;
5661 return Ok(());
5662 }
5663
5664 if self.state.foreign_checkout.is_some() {
5665 let wait = repository_wait(&self.state, None);
5668 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5669 if self.state.base_sync.is_none() {
5670 self.state.base_sync = Some(BaseSync {
5671 tip: tip.clone(),
5672 behind,
5673 attempts: 0,
5674 conflict: None,
5675 already_in: None,
5676 });
5677 }
5678 self.state.reserve_base_operation(&head, &tip);
5679 self.state.save()?;
5680 let merged = git::git_raw(
5681 &winner.worktree,
5682 &[
5683 "-c",
5684 "core.hooksPath=/dev/null",
5685 "merge",
5686 "--no-ff",
5687 "--no-autostash",
5688 "--no-edit",
5689 "-m",
5690 &format!("Merge {tracking} before verifying"),
5691 &tip,
5692 ],
5693 )
5694 .await?;
5695 let conflict = if merged.ok() {
5696 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5697 self.state.event(
5698 "land",
5699 format!("merged {tracking} into {} before verifying", winner.branch),
5700 );
5701 None
5702 } else {
5703 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5704 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5705 }
5706 let why = format!(
5707 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5708 winner.branch,
5709 merged.stderr.chars().take(600).collect::<String>()
5710 );
5711 self.state.status = RunStatus::Blocked;
5712 self.state.event("land", &why);
5713 Some(why)
5714 };
5715 let synced = conflict.is_none();
5716 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5717 self.state
5718 .finish_base_operation(synced.then_some(fresh.as_str()));
5719 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5720 self.state.base_sync = Some(BaseSync {
5721 tip,
5722 behind: if synced { 0 } else { behind },
5723 attempts,
5724 conflict,
5725 already_in: None,
5726 });
5727 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5728 .await?;
5729 self.state.save()?;
5730 return Ok(());
5731 }
5732
5733 self.state.event(
5734 "land",
5735 format!(
5736 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5737 winner.branch
5738 ),
5739 );
5740 self.state.save()?;
5741
5742 let branch_tracking = format!("{remote}/{}", winner.branch);
5748 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5749 let remote_tip = if self.state.task_pr_branch().is_none()
5750 && matches!(&fetched_branch, Ok(o) if o.ok())
5751 {
5752 git::rev_parse(&repo, &branch_tracking).await.ok()
5753 } else {
5754 None
5755 };
5756 if let Some(theirs) = &remote_tip
5760 && !git::is_ancestor(&repo, theirs, &head).await
5761 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5762 .await
5763 .is_ok_and(|missing| missing.is_empty())
5764 {
5765 let why = format!(
5766 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5767 them",
5768 short(theirs),
5769 winner.branch
5770 );
5771 self.state.status = RunStatus::Blocked;
5772 self.state.base_sync = Some(BaseSync {
5773 tip,
5774 behind,
5775 attempts,
5776 conflict: Some(why.clone()),
5777 already_in: None,
5778 });
5779 self.state.event("land", why);
5780 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5781 .await?;
5782 self.state.save()?;
5783 return Ok(());
5784 }
5785
5786 if self.state.base_sync.is_none() {
5787 self.state.base_sync = Some(BaseSync {
5788 tip: tip.clone(),
5789 behind,
5790 attempts: 0,
5791 conflict: None,
5792 already_in: None,
5793 });
5794 }
5795 self.state.reserve_base_operation(&head, &tip);
5796 self.state.save()?;
5797 let scratch = self.state.dir().join("base-sync");
5798 let rebased =
5799 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5800 .await
5801 {
5802 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5803 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5804 Err(e) => Err(e),
5805 };
5806 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5807 let stop_kind = match &rebased {
5808 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5809 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5810 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5811 };
5812 match rebased {
5813 Ok(None) => {
5814 git::sync_to_head(&winner.worktree).await?;
5818 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5819 self.state.finish_base_operation(Some(&fresh));
5820 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5821 let mut conflict = None;
5822 if let Some(pinned) = &remote_tip {
5823 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5824 match pushed {
5825 Ok(o) if o.ok() => self.state.event(
5826 "land",
5827 format!("pushed rebased {} to {remote}", winner.branch),
5828 ),
5829 Ok(o) => {
5830 conflict = Some(format!(
5831 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5832 winner.branch,
5833 short(pinned),
5834 o.stderr.chars().take(600).collect::<String>()
5835 ));
5836 }
5837 Err(e) => {
5838 conflict = Some(format!(
5839 "rebased {} locally but could not push it: {e:#}",
5840 winner.branch
5841 ));
5842 }
5843 }
5844 }
5845 if let Some(why) = &conflict {
5846 self.state.status = RunStatus::Blocked;
5847 self.state.event("land", why.clone());
5848 }
5849 self.state.base_sync = Some(BaseSync {
5850 tip: tip.clone(),
5851 behind: 0,
5852 attempts,
5853 conflict,
5854 already_in: None,
5855 });
5856 self.state
5857 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5858 }
5859 Ok(Some(conflict)) => {
5860 self.state.finish_base_operation(None);
5861 let why = format!(
5862 "{} conflicts with {tracking} and did not rebase: {}",
5863 winner.branch,
5864 conflict.chars().take(600).collect::<String>()
5865 );
5866 self.state.status = RunStatus::Blocked;
5867 self.state.base_sync = Some(BaseSync {
5868 tip,
5869 behind,
5870 attempts,
5871 conflict: Some(why.clone()),
5872 already_in: None,
5873 });
5874 self.state.event("land", why);
5875 }
5876 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5880 self.state.save()?;
5881 return Ok(());
5882 }
5883 Err(e) => {
5884 self.state.finish_base_operation(None);
5885 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5886 self.state.status = RunStatus::Blocked;
5887 self.state.base_sync = Some(BaseSync {
5888 tip,
5889 behind,
5890 attempts,
5891 conflict: Some(why.clone()),
5892 already_in: None,
5893 });
5894 self.state.event("land", why);
5895 }
5896 }
5897 if !self.state.base_sync_blocked() {
5898 self.state.base_sync_recovery.stop = None;
5899 }
5900 self.hold_base_sync_kind(stop_kind).await?;
5901 self.state.save()?;
5902 Ok(())
5903 }
5904
5905 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5911 if self.state.task_pr_branch().is_some() {
5914 return None;
5915 }
5916 let repo = self.state.repo.clone();
5917 let remote = self.state.config.merge.remote.clone();
5918 let fetched = git::fetch(&repo, &remote, branch).await;
5919 if !matches!(&fetched, Ok(o) if o.ok()) {
5920 return None;
5921 }
5922 let branch_tracking = format!("{remote}/{branch}");
5923 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5924 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5925 return None;
5926 }
5927 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5928 .await
5929 .is_ok_and(|missing| missing.is_empty())
5930 {
5931 return Some(format!(
5932 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5933 them",
5934 short(&theirs)
5935 ));
5936 }
5937 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5938 Ok(o) if o.ok() => {
5939 self.state
5940 .event("land", format!("pushed rebased {branch} to {remote}"));
5941 None
5942 }
5943 Ok(o) => Some(format!(
5944 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5945 someone may have pushed): {}",
5946 short(&theirs),
5947 o.stderr.chars().take(600).collect::<String>()
5948 )),
5949 Err(e) => Some(format!(
5950 "{branch} is rebased locally but could not be pushed: {e:#}"
5951 )),
5952 }
5953 }
5954
5955 async fn settle_already_in(
5972 &mut self,
5973 branch: &str,
5974 tip: &str,
5975 head: &str,
5976 attempts: usize,
5977 behind: usize,
5978 ) -> Result<bool> {
5979 let repo = self.state.repo.clone();
5980 let remote = self.state.config.merge.remote.clone();
5981 let start = self.state.base_commit.clone();
5982 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5983 Ok(Some(e)) => e,
5984 Ok(None) => return Ok(false),
5985 Err(e) => {
5986 tracing::warn!("already-in-base check for {branch}: {e:#}");
5987 return Ok(false);
5988 }
5989 };
5990 let mut verified = vec![head.to_owned()];
5991 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5992 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5993 if matches!(&fetched, Ok(o) if o.ok())
5994 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5995 && theirs != head
5996 {
5997 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5998 Ok(Some(_)) => verified.push(theirs),
5999 _ => return Ok(false),
6000 }
6001 }
6002 let base_branch = self.state.base_branch.clone();
6003 let message = format!(
6004 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6005 land",
6006 evidence.names(),
6007 evidence.proof.as_str()
6008 );
6009 let closed =
6010 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6011 .await;
6012 match closed {
6013 Ok(Ok(url)) => self
6014 .state
6015 .event("land", format!("closed {url}: superseded on {base_branch}")),
6016 Ok(Err(why)) => self
6017 .state
6018 .event("land", format!("did not close a pull request: {why}")),
6019 Err(e) => {
6020 let why = format!(
6021 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6022 not be closed ({e:#}); resume to retry"
6023 );
6024 self.state.status = RunStatus::Blocked;
6025 self.state.base_sync = Some(BaseSync {
6026 tip: tip.to_owned(),
6027 behind,
6028 attempts,
6029 conflict: Some(why.clone()),
6030 already_in: None,
6031 });
6032 self.state.event("land", why);
6033 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6034 .await?;
6035 self.state.save()?;
6036 return Ok(true);
6037 }
6038 }
6039 self.state.status = RunStatus::AlreadyInBase;
6040 self.state.base_sync_recovery.stop = None;
6041 self.state.base_sync = Some(BaseSync {
6042 tip: tip.to_owned(),
6043 behind,
6044 attempts,
6045 conflict: None,
6046 already_in: Some(evidence),
6047 });
6048 self.state.event("land", message);
6049 self.state.save()?;
6050 self.settle_questions();
6051 Ok(true)
6052 }
6053
6054 fn landing_base(&self) -> String {
6064 self.state
6065 .base_sync
6066 .as_ref()
6067 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6068 }
6069
6070 pub async fn fix_selected(
6103 &mut self,
6104 ids: &[String],
6105 reason: &str,
6106 allow_stale: bool,
6107 ) -> Result<()> {
6108 let reason = reason.trim();
6109 if reason.is_empty() {
6110 bail!("a fix request needs a reason — that is the operator's own record of why");
6111 }
6112 if ids.is_empty() {
6113 bail!("no finding id given");
6114 }
6115 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6116 bail!(
6117 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6118 has already concluded — can be given a targeted fix. A run still \
6119 in progress should simply be resumed; a `merged` run's branch has \
6120 already landed, so its answer is a fresh `magi review <branch>`, \
6121 not reopening this run's own record",
6122 self.state.id,
6123 self.state.status.as_str()
6124 );
6125 }
6126 let Some(winner) = self.state.winner().cloned() else {
6127 bail!("run {} has no winning candidate to fix", self.state.id);
6128 };
6129 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6130 bail!(
6131 "branch `{}` no longer exists; this run cannot be extended",
6132 winner.branch
6133 );
6134 }
6135 let home = crate::run::home();
6136 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6137 bail!(
6138 "run {} is currently being worked on by another magi process",
6139 self.state.id
6140 );
6141 }
6142 let _claim = FixClaim::acquire(&self.state.dir())?;
6148
6149 let mut seen = BTreeSet::new();
6153 let mut findings = Vec::new();
6154 let mut missing = Vec::new();
6155 for id in ids {
6156 if !seen.insert(id.clone()) {
6157 continue;
6158 }
6159 match self.state.finding(id) {
6160 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6161 id: f.id.clone(),
6162 severity: f.severity,
6163 reviewer_vote: rec.vote,
6164 round: round.round,
6165 round_head: round.head.clone(),
6166 reviewer: rec.reviewer,
6167 agent: rec.agent.clone(),
6168 file: f.file.clone(),
6169 line: f.line,
6170 title: f.title.clone(),
6171 detail: f.detail.clone(),
6172 outcome: OperatorFixOutcome::Pending,
6173 }),
6174 None => missing.push(id.clone()),
6175 }
6176 }
6177 if !missing.is_empty() {
6178 bail!(
6179 "unknown finding id(s): {}; nothing was changed",
6180 missing.join(", ")
6181 );
6182 }
6183
6184 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6185 let stale_details: Vec<(String, String)> = findings
6186 .iter()
6187 .filter(|f| f.round_head != head_at_request)
6188 .map(|f| (f.id.clone(), f.round_head.clone()))
6189 .collect();
6190 let stale = !stale_details.is_empty();
6191 if stale && !allow_stale {
6192 bail!(
6193 "the branch has moved since some finding(s) were raised — {} — now \
6194 at {}; pass --allow-stale to fix anyway, or re-run review first",
6195 stale_details
6196 .iter()
6197 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6198 .collect::<Vec<_>>()
6199 .join(", "),
6200 short(&head_at_request)
6201 );
6202 }
6203
6204 let request = OperatorFixRequest {
6205 requested_at: Timestamp::now(),
6206 reason: reason.to_owned(),
6207 findings,
6208 head_at_request: head_at_request.clone(),
6209 allow_stale,
6210 stale,
6211 fix: None,
6212 result_head: None,
6213 follow_up_review_run: None,
6214 };
6215 self.state.event(
6216 "fix",
6217 format!(
6218 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6219 request.findings.len(),
6220 request
6221 .findings
6222 .iter()
6223 .map(|f| f.id.as_str())
6224 .collect::<Vec<_>>()
6225 .join(", "),
6226 ),
6227 );
6228 self.state.operator_fixes.push(request);
6235 self.state.save()?;
6236 let request_index = self.state.operator_fixes.len() - 1;
6237
6238 let repo = self.state.repo.clone();
6247 let coordination = self.coordinate(&repo).await?;
6248 if winner.worktree.exists() {
6249 let dirty = git::git(
6252 &winner.worktree,
6253 &["status", "--porcelain", "--untracked-files=all"],
6254 )
6255 .await?;
6256 let only_withheld = dirty.lines().all(|l| {
6257 l.strip_prefix("?? ")
6258 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6259 });
6260 if !only_withheld {
6261 bail!(
6262 "`{}` has uncommitted changes; refusing to touch it — commit or \
6263 discard them first",
6264 winner.worktree.display()
6265 );
6266 }
6267 git::worktree_remove(&self.state.repo, &winner.worktree)
6268 .await
6269 .ok();
6270 }
6271 let fix_worktree = self.state.worktree_root().join("operator-fix");
6272 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6273 git::git(
6274 &self.state.repo,
6275 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6276 )
6277 .await
6278 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6279 if !git::is_clean(&fix_worktree).await? {
6280 git::worktree_remove(&self.state.repo, &fix_worktree)
6281 .await
6282 .ok();
6283 bail!(
6284 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6285 winner.branch
6286 );
6287 }
6288 let repo_for_setup = self.state.repo.clone();
6289 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6290 Ok(()) => {
6291 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6292 }
6293 Err(e) => Err(e),
6294 };
6295 if let Err(e) = setup {
6296 git::worktree_remove(&self.state.repo, &fix_worktree)
6297 .await
6298 .ok();
6299 return Err(e);
6300 }
6301
6302 self.state.resource_block = None;
6305
6306 let run_id = self.state.id.clone();
6307 let prompts = self.state.config.prompts.clone();
6308 let language = self.state.config.graph.language.clone();
6309 let sessions = self.state.config.graph.sessions;
6310 let artifacts = agent::artifacts_dir(&self.state.dir());
6311 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6312 .findings
6313 .iter()
6314 .map(|f| Finding {
6315 id: f.id.clone(),
6316 severity: f.severity,
6317 file: f.file.clone(),
6318 line: f.line,
6319 title: f.title.clone(),
6320 detail: f.detail.clone(),
6321 })
6322 .collect();
6323 let fix_prompt = prompt::operator_fix_in(
6324 &self.state.instruction,
6325 &finding_list,
6326 reason,
6327 &stale_details,
6328 &head_at_request,
6329 &language,
6330 &self.state.config.graph.github_language,
6331 );
6332 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6333 drop(coordination);
6334 let (job, seat, out) = self
6335 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6336 prompt: fix_prompt.clone(),
6337 spec,
6338 seat,
6339 cwd: fix_worktree.clone(),
6340 timeout,
6341 allow_write: true,
6342 sessions,
6343 artifacts: artifacts.clone(),
6344 stem: "operator-fix".to_owned(),
6345 handover: None,
6346 })
6347 .await;
6348 let agent_id = seat.agent.clone();
6349
6350 let mut fix = FixRecord {
6351 agent: agent_id,
6352 addressed: Vec::new(),
6353 rejected: Vec::new(),
6354 notes: String::new(),
6355 committed: false,
6356 failed: None,
6357 duration_ms: 0,
6358 continuation: None,
6359 };
6360 let mut final_seat = seat.clone();
6361 match out {
6362 AgentOutcome::Ok(o) => {
6363 fix.duration_ms = o.duration_ms;
6364 let parsed = verdict::extract_json::<FixReport>(&o.text);
6365 let incomplete_reason = match &parsed {
6366 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6367 "the reply parsed, but it reported a command whose own CLI \
6368 never confirmed an exit status"
6369 .to_owned(),
6370 ),
6371 Ok(_) => None,
6372 Err(e) => Some(e.to_string()),
6373 };
6374 match incomplete_reason {
6375 None => {
6376 let report = parsed.expect("checked Ok above");
6377 fix.addressed = report.addressed;
6378 fix.rejected = report.rejected;
6379 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6380 }
6381 Some(reason) => {
6382 let (resumed_seat, resolved, failure, cont) = self
6383 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6384 .await;
6385 fix.duration_ms += cont.cumulative_wait_ms;
6386 fix.continuation = Some(cont);
6387 final_seat = resumed_seat;
6388 match resolved {
6389 Some(report) => {
6390 fix.addressed = report.addressed;
6391 fix.rejected = report.rejected;
6392 fix.notes =
6393 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6394 }
6395 None => fix.failed = failure,
6396 }
6397 }
6398 }
6399 }
6400 AgentOutcome::Dropped(o) => {
6401 fix.duration_ms = o.duration_ms;
6402 let why = o
6403 .dropped
6404 .as_ref()
6405 .map(|d| d.why.as_str())
6406 .unwrap_or("the CLI ended the stream without delivering its answer");
6407 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6408 }
6409 AgentOutcome::Quota(o) => {
6410 self.state.quota.push(QuotaLoss::from_report(
6411 final_seat.key.clone(),
6412 "fix",
6413 o.quota.as_ref(),
6414 ));
6415 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6416 }
6417 AgentOutcome::Failed(e) => fix.failed = Some(e),
6418 AgentOutcome::ResourceBlocked(e) => {
6419 fix.failed = Some(format!("the fixer was not started: {e}"));
6420 }
6421 }
6422 if fix.continuation.is_none() {
6425 fix.continuation = Some(ContinuationRecord::not_needed());
6426 }
6427 self.state.seats.insert(final_seat.key.clone(), final_seat);
6428
6429 let rescue_message = format!(
6430 "magi: operator-selected fix ({}) (uncommitted work)",
6431 self.state.operator_fixes[request_index]
6432 .findings
6433 .iter()
6434 .map(|f| f.id.as_str())
6435 .collect::<Vec<_>>()
6436 .join(", ")
6437 );
6438 let repo = self.state.repo.clone();
6439 let coordination = self.coordinate(&repo).await?;
6440 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6441 self.state.note_withheld("fix", &r.withheld);
6442 }
6443 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6444 fix.committed = after != head_at_request;
6445 git::worktree_remove(&self.state.repo, &fix_worktree)
6446 .await
6447 .ok();
6448 drop(coordination);
6449
6450 self.state.event(
6451 "fix",
6452 match &fix.failed {
6453 Some(reason) => format!(
6454 "operator fix: adoption report was lost ({reason}); {}",
6455 if fix.committed {
6456 "committed"
6457 } else {
6458 "NO new commit"
6459 }
6460 ),
6461 None => format!(
6462 "operator fix: {} addressed, {} rejected, {}",
6463 fix.addressed.len(),
6464 fix.rejected.len(),
6465 if fix.committed {
6466 "committed"
6467 } else {
6468 "NO new commit"
6469 }
6470 ),
6471 },
6472 );
6473
6474 for f in &mut self.state.operator_fixes[request_index].findings {
6481 f.outcome = if fix.failed.is_some() {
6482 OperatorFixOutcome::Unreported
6483 } else if fix.addressed.contains(&f.id) {
6484 OperatorFixOutcome::Addressed
6485 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6486 OperatorFixOutcome::Rejected { why: r.why.clone() }
6487 } else {
6488 OperatorFixOutcome::Unreported
6489 };
6490 }
6491
6492 let committed = fix.committed;
6493 if committed {
6494 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6495 }
6496 self.state.operator_fixes[request_index].fix = Some(fix);
6497 self.state.save()?;
6500 if self.state.resource_block.is_some() {
6501 return self.stop_for_resource();
6502 }
6503
6504 if committed {
6505 self.state.event(
6506 "fix",
6507 format!(
6508 "operator fix committed {}; opening a follow-up review-only run",
6509 short(&after)
6510 ),
6511 );
6512 let origin =
6515 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6516 match Self::review(
6517 &self.state.repo,
6518 &winner.branch,
6519 self.state.config.clone(),
6520 origin,
6521 )
6522 .await
6523 {
6524 Ok(mut follow_up) => {
6525 follow_up.state.event(
6526 "start",
6527 format!(
6528 "requested by an operator fix on run {} for finding(s) {}",
6529 self.state.id,
6530 self.state.operator_fixes[request_index]
6531 .findings
6532 .iter()
6533 .map(|f| f.id.as_str())
6534 .collect::<Vec<_>>()
6535 .join(", "),
6536 ),
6537 );
6538 follow_up.state.save()?;
6539 let follow_up_id = follow_up.state.id.clone();
6540 let adopted = match crate::direct::adopt(
6543 &follow_up.state,
6544 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6545 ) {
6546 Ok(a) => a,
6547 Err(e) => {
6548 self.state.event(
6551 "fix",
6552 format!(
6553 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6554 ),
6555 );
6556 self.state.save()?;
6557 return Ok(());
6558 }
6559 };
6560 let executed = follow_up.execute().await;
6561 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6562 if let Some(a) = adopted {
6563 a.finish(&follow_up.state, executed);
6564 }
6565 if let Some(e) = failure {
6566 self.state.event(
6567 "fix",
6568 format!(
6569 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6570 ),
6571 );
6572 }
6573 self.state.operator_fixes[request_index].follow_up_review_run =
6574 Some(follow_up_id);
6575 }
6576 Err(e) => {
6577 self.state.event(
6578 "fix",
6579 format!("committed the fix but could not open a follow-up review: {e:#}"),
6580 );
6581 }
6582 }
6583 self.state.save()?;
6584 }
6585
6586 Ok(())
6587 }
6588
6589 async fn ask_fixer(
6609 &mut self,
6610 winner: &Candidate,
6611 node: &'static str,
6612 round: Option<usize>,
6613 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6614 ) -> (SeatJob, SeatState, AgentOutcome) {
6615 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6616 self.ask_fixer_among(attempts, node, round, None, build)
6617 .await
6618 }
6619
6620 async fn ask_fixer_among(
6625 &mut self,
6626 attempts: Vec<(AgentSpec, String)>,
6627 node: &'static str,
6628 round: Option<usize>,
6629 restore: Option<(&Path, &str)>,
6630 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6631 ) -> (SeatJob, SeatState, AgentOutcome) {
6632 let run_id = self.state.id.clone();
6633 let prompts = self.state.config.prompts.clone();
6634 let cache = self.state.config.cache_dir();
6635 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6636 let mut last = None;
6637 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6638 let seat = self.seat(&key, &spec.id);
6639 let mut job = build(spec.clone(), seat);
6640 if i > 0 {
6641 job.stem = format!("{}-{}", job.stem, spec.id);
6642 }
6643 let ctx = WaveCtx {
6644 carry_seats: false,
6645 pauses: Some((&self.pause, &self.interrupt)),
6646 run: &run_id,
6647 node,
6648 prompts: &prompts,
6649 cache: cache.as_deref(),
6650 round,
6651 };
6652 let (seat, out) =
6653 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6654 let advances = match &out {
6657 AgentOutcome::Ok(o) => agent::output_advances(o),
6658 AgentOutcome::ResourceBlocked(_) => false,
6659 _ => true,
6660 };
6661 if let Some(next) = ids.get(i + 1)
6662 && advances
6663 {
6664 self.state.seats.insert(seat.key.clone(), seat.clone());
6665 let class =
6666 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6667 let reason = match &out {
6668 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6669 other => fail_reason(other),
6670 };
6671 if let Some((wt, rev)) = restore
6675 && let Err(e) = self.graft_restore(wt, rev).await
6676 {
6677 let why =
6678 format!("could not restore the worktree before the next fixer: {e:#}");
6679 tracing::warn!("{why}");
6680 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6681 AgentOutcome::ResourceBlocked(why)
6682 } else {
6683 AgentOutcome::Failed(why)
6684 };
6685 last = Some((job, seat, out));
6686 break;
6687 }
6688 record_handover(
6689 &mut self.state,
6690 node,
6691 &seat.key,
6692 &spec.id,
6693 next,
6694 &class,
6695 &reason,
6696 );
6697 continue;
6698 }
6699 last = Some((job, seat, out));
6700 break;
6701 }
6702 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6703 if let AgentOutcome::Quota(output) = &out
6704 && node != "graft"
6705 {
6706 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6711 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6712 self.state.quota.push(loss);
6713 self.state.seats.insert(seat.key.clone(), seat.clone());
6714 self.state.status = RunStatus::Blocked;
6715 return (
6716 job,
6717 seat,
6718 AgentOutcome::ResourceBlocked(
6719 "fixer unavailable; restore agent availability and resume".into(),
6720 ),
6721 );
6722 }
6723 (job, seat, out)
6724 }
6725
6726 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6737 use crate::recovery::{Mode, Repair};
6738 if !self.state.reviews.is_empty() {
6739 return Ok(());
6740 }
6741 let Some(mut rec) = self.state.recovery.clone() else {
6742 return Ok(());
6743 };
6744 if rec.mode != Mode::VerifyRepair {
6745 return Ok(());
6746 }
6747 let wt = winner.worktree.clone();
6748 let mut head = git::rev_parse(&wt, "HEAD").await?;
6749 match rec.repair.clone() {
6750 Repair::Done { .. } => return Ok(()),
6751 Repair::WaitingForRestore { before } => {
6752 self.graft_restore(&wt, &before).await?;
6753 head = before;
6754 rec.repair = Repair::NotStarted;
6755 self.state.recovery = Some(rec.clone());
6756 self.state.save()?;
6757 }
6758 Repair::Started { before } => {
6759 if let Err(e) = self.graft_restore(&wt, &before).await {
6763 if e.is::<crate::repo_lock::ResourceBlocked>() {
6764 return Err(e);
6765 }
6766 return self
6767 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6768 .await;
6769 }
6770 rec.repair = Repair::Done {
6771 before: before.clone(),
6772 after: before,
6773 committed: false,
6774 failed: Some("interrupted; rolled back".to_owned()),
6775 };
6776 self.state
6777 .event("recovery", "the repair was interrupted and rolled back");
6778 self.state.recovery = Some(rec);
6779 self.state.save()?;
6780 return Ok(());
6781 }
6782 Repair::NotStarted => {}
6783 }
6784 if crate::recovery::recheck(&mut rec, &head) {
6785 self.state.event("recovery", rec.summary());
6786 self.state.recovery = Some(rec);
6787 self.state.save()?;
6788 return Ok(());
6789 }
6790 rec.repair = Repair::Started {
6791 before: head.clone(),
6792 };
6793 self.state.recovery = Some(rec.clone());
6794 self.state.save()?;
6795
6796 let language = self.state.config.graph.language.clone();
6797 let sessions = self.state.config.graph.sessions;
6798 let artifacts = agent::artifacts_dir(&self.state.dir());
6799 let fix_prompt = prompt::recovery_fix(
6800 &self.state.instruction,
6801 &rec,
6802 self.state.config.graph.review_rounds,
6803 &language,
6804 );
6805 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6806 self.state.event(
6807 "recovery",
6808 format!(
6809 "repair: the fixer starts from the failed verification on {}",
6810 short(&head)
6811 ),
6812 );
6813 let (_job, seat, out) = self
6814 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6815 prompt: fix_prompt.clone(),
6816 spec,
6817 seat,
6818 cwd: wt.clone(),
6819 timeout,
6820 allow_write: true,
6821 sessions,
6822 artifacts: artifacts.clone(),
6823 stem: "fix-recovery".to_owned(),
6824 handover: None,
6825 })
6826 .await;
6827 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6828 rec.repair = Repair::WaitingForRestore { before: head };
6829 self.state.recovery = Some(rec);
6830 return self.stop_for_resource();
6831 }
6832 let failed = match &out {
6833 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6834 AgentOutcome::Quota(o) => {
6835 self.state.quota.push(QuotaLoss::from_report(
6836 seat.key.clone(),
6837 "fix",
6838 o.quota.as_ref(),
6839 ));
6840 Some("rate limited (quota); the fixer could not run".to_owned())
6841 }
6842 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6843 AgentOutcome::Failed(e) => Some(e.clone()),
6844 };
6845 self.state.seats.insert(seat.key.clone(), seat);
6846 let coordination = self.coordinate(&wt).await?;
6847 let failed = match failed {
6848 Some(why) => {
6850 if let Err(e) = git::restore_to(&wt, &head).await {
6851 return self
6852 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6853 .await;
6854 }
6855 Some(why)
6856 }
6857 None => {
6858 if let Ok(r) = git::rescue_commit(
6859 &wt,
6860 "magi: recovery repair of the failed verification (uncommitted work)",
6861 )
6862 .await
6863 {
6864 self.state.note_withheld("fix", &r.withheld);
6865 }
6866 None
6867 }
6868 };
6869 let after = git::rev_parse(&wt, "HEAD").await?;
6870 drop(coordination);
6871 self.state.event(
6872 "recovery",
6873 match &failed {
6874 Some(why) => format!("repair: nothing kept ({why})"),
6875 None if after != head => {
6876 format!("repair: committed {}", short(&after))
6877 }
6878 None => "repair: the fixer made no commit".to_owned(),
6879 },
6880 );
6881 rec.repair = Repair::Done {
6882 before: head.clone(),
6883 committed: after != head,
6884 after,
6885 failed,
6886 };
6887 self.state.recovery = Some(rec);
6888 self.state.save()?;
6889 Ok(())
6890 }
6891
6892 async fn stop_recovery(
6895 &mut self,
6896 mut rec: crate::recovery::RecoveryRecord,
6897 head: &str,
6898 why: String,
6899 ) -> Result<()> {
6900 rec.repair = crate::recovery::Repair::Started {
6902 before: head.to_owned(),
6903 };
6904 self.state.recovery = Some(rec);
6905 self.state
6906 .event("recovery", format!("repair could not be undone: {why}"));
6907 self.state.status = RunStatus::Blocked;
6908 self.state.save()?;
6909 bail!("recovery repair could not be rolled back: {why}")
6910 }
6911
6912 async fn review_loop(&mut self) -> Result<()> {
6913 if self.hold_base_sync().await? {
6914 return Ok(());
6915 }
6916 loop {
6917 if self
6918 .state
6919 .base_sync
6920 .as_ref()
6921 .is_some_and(|s| s.conflict.is_some())
6922 {
6923 return Ok(());
6924 }
6925 if self.catchup_repairable() {
6926 let record = self.state.reviews.last().expect("repairable round").clone();
6927 let winner = self
6928 .state
6929 .winner()
6930 .cloned()
6931 .context("catch-up repair has no winner")?;
6932 if !self.fix_review_round(&winner, record).await? {
6933 return Ok(());
6934 }
6935 }
6936 self.review_loop_pass().await?;
6937 if !self.catchup_repairable() {
6938 return Ok(());
6939 }
6940 }
6941 }
6942
6943 fn catchup_repairable(&self) -> bool {
6946 self.state.reviews.len() < self.state.config.graph.review_rounds
6947 && self.state.reviews.last().is_some_and(|r| {
6948 !r.clean
6949 && r.blocking == 0
6950 && r.fix.is_none()
6951 && r.e2e_status() == E2eStatus::Failed
6952 })
6953 }
6954
6955 async fn review_loop_pass(&mut self) -> Result<()> {
6956 if self
6961 .state
6962 .base_sync
6963 .as_ref()
6964 .is_some_and(|s| s.conflict.is_some())
6965 {
6966 return Ok(());
6967 }
6968 let run_id = self.state.id.clone();
6973 let prompts = self.state.config.prompts.clone();
6974 let Some(winner) = self.state.winner().cloned() else {
6975 return Ok(());
6976 };
6977 let granted_review = self
6978 .state
6979 .repair_grants
6980 .last()
6981 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6982 .cloned();
6983 let fresh_review = granted_review
6984 .as_ref()
6985 .is_some_and(|g| self.state.reviews.len() == g.review_start);
6986 let max_rounds = granted_review
6987 .as_ref()
6988 .map_or(self.state.config.graph.review_rounds, |g| {
6989 g.review_start.saturating_add(1)
6990 });
6991 if let Some(status) =
7001 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7002 {
7003 if status == RunStatus::Gating {
7006 self.record_contested_handoff();
7007 }
7008 self.state.status = status;
7009 self.state.save()?;
7010 return Ok(());
7011 }
7012 self.state.status = RunStatus::Reviewing;
7013 if !fresh_review
7023 && self.state.reviews.last().is_some_and(|r| {
7024 r.e2e_status() == E2eStatus::ResourceBlocked
7025 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7026 })
7027 {
7028 let shell = self.state.config.shell();
7029 return self
7030 .stop_unresolved_review(
7031 "the last round's own verification never resolved",
7032 &shell,
7033 &winner.worktree,
7034 )
7035 .await;
7036 }
7037
7038 if granted_review.is_none() {
7039 self.recovery_repair(&winner).await?;
7040 }
7041 if self.resource_halt()? {
7042 return Ok(());
7043 }
7044
7045 let repo = self.state.repo.clone();
7046 let root = self.state.worktree_root();
7047 let language = self.state.config.graph.language.clone();
7048 let sessions = self.state.config.graph.sessions;
7049 let artifacts = self.merge_repair_artifacts();
7050 let base = self.landing_base();
7051 let base_short = short(&base);
7052 let reviewers = self.roles.reviewers.clone();
7053 let shell = self.state.config.shell();
7054
7055 for round in (self.state.reviews.len() + 1)..=max_rounds {
7056 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7057 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7058 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7059 let prev_verification = self
7068 .state
7069 .reviews
7070 .last()
7071 .and_then(|r| r.verification_summary(&head));
7072 let prior_section = match (&self.state.recovery, round) {
7075 (Some(rec), 1) if self.state.reviews.is_empty() => {
7076 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7077 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7078 Some(crate::recovery::head_of(
7079 &git::diff(&winner.worktree, reviewed, "HEAD")
7080 .await
7081 .unwrap_or_default(),
7082 crate::recovery::SINCE_DIFF_MAX,
7083 ))
7084 }
7085 _ => None,
7086 };
7087 Some(prompt::recovery_review(rec, since.as_deref()))
7088 }
7089 _ => None,
7090 };
7091
7092 let mut jobs = Vec::new();
7096 let coordination = self.coordinate(&repo).await?;
7097 for (r, spec) in reviewers.iter().cloned().enumerate() {
7098 let wt = root.join(format!("review-{}", r + 1));
7099 if wt.exists() {
7102 git::reset_detached(&wt, &head).await?;
7103 } else {
7104 git::worktree_add_detached(&repo, &wt, &head).await?;
7105 }
7106 self.setup_seat_worktree(&repo, &wt, false).await?;
7107 let seat_key = format!("review-{}", r + 1);
7108 let spec = pick_start_spec(
7111 &self.roles.reviewer_roster,
7112 spec,
7113 self.state.seat_history.get(&seat_key),
7114 );
7115 let seat = self.seat(&seat_key, &spec.id);
7116 jobs.push(SeatJob {
7117 prompt: prompt::review(&prompt::ReviewCtx {
7118 instruction: &self.state.instruction,
7119 branch: &winner.branch,
7120 base_short: &base_short,
7121 stat: &stat,
7122 patch: &patch,
7123 verification: prev_verification.as_ref(),
7124 prior: prior_section.as_deref(),
7125 reviewers: reviewers.len(),
7126 round,
7127 rounds: max_rounds,
7128 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7131 lens: Lens::for_seat(r),
7132 language: &language,
7133 github_language: &self.state.config.graph.github_language,
7134 }),
7135 spec,
7136 seat,
7137 cwd: wt,
7138 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7139 allow_write: false,
7140 sessions,
7141 artifacts: artifacts.clone(),
7142 stem: format!("review-{round}-{}", r + 1),
7143 handover: None,
7144 });
7145 }
7146
7147 drop(coordination);
7148 self.state.event(
7149 "review",
7150 format!(
7151 "round {round}: {} reviewers on {}",
7152 jobs.len(),
7153 short(&head)
7154 ),
7155 );
7156 let mut quota_losses = Vec::new();
7157 let review_retries = self.state.config.graph.retries;
7158 let review_cache = self.state.config.cache_dir();
7159 let ctx = WaveCtx {
7160 carry_seats: true,
7161 pauses: Some((&self.pause, &self.interrupt)),
7162 run: &run_id,
7163 node: "review",
7164 prompts: &prompts,
7165 cache: review_cache.as_deref(),
7166 round: Some(round),
7167 };
7168 let results = ask_json_wave::<Review>(
7169 jobs,
7170 Arc::clone(&self.sem),
7171 review_retries,
7172 &self.roles.reviewer_roster,
7173 &ctx,
7174 &mut quota_losses,
7175 &mut self.state,
7176 &|_: &Review| Ok(()),
7177 )
7178 .await;
7179 if self.state.resource_block.is_some() {
7180 return self.stop_for_resource();
7181 }
7182 let round_quota_missing = quota_losses.len();
7186 self.state.quota.extend(quota_losses);
7187
7188 let mut records = Vec::new();
7189 let mut all_findings = Vec::new();
7190 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7191 let agent_id = seat.agent.clone();
7192 self.state.seats.insert(seat.key.clone(), seat);
7193 let mut record = ReviewRecord {
7194 reviewer: r + 1,
7195 agent: agent_id,
7196 summary: String::new(),
7197 findings: Vec::new(),
7198 vote: None,
7199 failed: None,
7200 duration_ms: 0,
7201 attempts,
7207 };
7208 match res {
7209 Ok((review, out)) => {
7210 record.summary =
7218 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7219 record.vote = Some(review.vote);
7220 record.duration_ms = out.duration_ms;
7221 for (n, mut f) in review.findings.into_iter().enumerate() {
7222 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7225 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7226 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7227 f.file = f
7233 .file
7234 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7235 all_findings.push(f.clone());
7236 record.findings.push(f);
7237 }
7238 self.state.event(
7239 "review",
7240 format!(
7241 "round {round}: reviewer {} voted {} with {} finding(s)",
7242 r + 1,
7243 review.vote.label(),
7244 record.findings.len()
7245 ),
7246 );
7247 }
7248 Err(e) => {
7249 record.failed = Some(e.to_string());
7250 self.state.event(
7251 "review",
7252 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7253 );
7254 }
7255 }
7256 records.push(record);
7257 }
7258
7259 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7266 let vote_split =
7267 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7268 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7269 if vote_split {
7270 self.state.event(
7271 "review",
7272 format!(
7273 "round {round}: votes split ({}) — one round of reconsideration",
7274 initial_votes
7275 .iter()
7276 .map(|v| v.label())
7277 .collect::<Vec<_>>()
7278 .join(", ")
7279 ),
7280 );
7281 let panel: Vec<ReviewSeatReport<'_>> = records
7284 .iter()
7285 .filter_map(|r| {
7286 r.vote.map(|vote| ReviewSeatReport {
7287 reviewer: r.reviewer,
7288 vote,
7289 summary: &r.summary,
7290 findings: &r.findings,
7291 })
7292 })
7293 .collect();
7294
7295 let mut jobs = Vec::new();
7296 let mut seats_at = Vec::new();
7297 for (r, spec) in reviewers.iter().cloned().enumerate() {
7298 if records[r].vote.is_none() {
7302 continue;
7303 }
7304 let wt = root.join(format!("review-{}", r + 1));
7305 let seat_key = format!("review-{}", r + 1);
7306 let spec = self.occupant(&seat_key, spec);
7307 let seat = self.seat(&seat_key, &spec.id);
7308 let build = |with_patch: bool| {
7316 prompt::review_reconsider(&ReviewReconsiderCtx {
7317 instruction: &self.state.instruction,
7318 reviewer: r + 1,
7319 lens: Lens::for_seat(r),
7320 panel: &panel,
7321 patch: with_patch.then_some(ReviewPatch {
7322 branch: &winner.branch,
7323 base_short: &base_short,
7324 stat: &stat,
7325 patch: &patch,
7326 }),
7327 round,
7328 rounds: max_rounds,
7329 language: &language,
7330 })
7331 };
7332 let full = build(true);
7333 let prompt = if has_context(&spec, &seat, sessions) {
7334 build(false)
7335 } else {
7336 full.clone()
7337 };
7338 jobs.push(SeatJob {
7339 prompt,
7340 spec,
7341 seat,
7342 cwd: wt,
7343 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7344 allow_write: false,
7345 sessions,
7346 artifacts: artifacts.clone(),
7347 stem: format!("review-{round}-reconsider-{}", r + 1),
7348 handover: Some(full),
7349 });
7350 seats_at.push(r);
7351 }
7352
7353 let mut recon_quota_losses = Vec::new();
7354 let recon_cache = self.state.config.cache_dir();
7355 let recon_ctx = WaveCtx {
7356 carry_seats: true,
7357 pauses: Some((&self.pause, &self.interrupt)),
7358 run: &run_id,
7359 node: "review",
7360 prompts: &prompts,
7361 cache: recon_cache.as_deref(),
7362 round: Some(round),
7363 };
7364 let recon_results = ask_json_wave::<ReviewRevote>(
7365 jobs,
7366 Arc::clone(&self.sem),
7367 review_retries,
7368 &self.roles.reviewer_roster,
7369 &recon_ctx,
7370 &mut recon_quota_losses,
7371 &mut self.state,
7372 &|_: &ReviewRevote| Ok(()),
7373 )
7374 .await;
7375 if self.state.resource_block.is_some() {
7376 return self.stop_for_resource();
7377 }
7378 self.state.quota.extend(recon_quota_losses);
7379
7380 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7381 let agent_id = seat.agent.clone();
7382 self.state.seats.insert(seat.key.clone(), seat);
7383 let mut rec = ReviewRevoteRecord {
7384 reviewer: r + 1,
7385 agent: agent_id,
7386 vote: None,
7387 reason: String::new(),
7388 failed: None,
7389 };
7390 match res {
7391 Ok((rv, _)) => {
7392 rec.vote = Some(rv.vote);
7393 rec.reason =
7394 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7395 self.state.event(
7396 "review",
7397 format!(
7398 "round {round}: reviewer {} revoted {}",
7399 r + 1,
7400 rv.vote.label()
7401 ),
7402 );
7403 }
7404 Err(e) => {
7405 rec.failed = Some(e.to_string());
7406 self.state.event(
7407 "review",
7408 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7409 );
7410 }
7411 }
7412 reconsideration.push(rec);
7413 }
7414 } else if initial_votes.len() > 1 {
7415 self.state.event(
7416 "review",
7417 format!(
7418 "round {round}: votes agreed ({}) — no reconsideration",
7419 initial_votes[0].label()
7420 ),
7421 );
7422 }
7423
7424 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7425 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7426 let defer_e2e =
7437 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7438 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7439 let reason =
7440 format!("{blocking} blocking finding(s) already required a fix this round");
7441 self.state.event(
7442 "verify",
7443 format!(
7444 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7445 {}); it will run once a round has none left",
7446 short(&head)
7447 ),
7448 );
7449 (Vec::new(), false, true, Some(reason))
7450 } else {
7451 let e2e_commands = self.state.config.verify.e2e.clone();
7452 let cache_dir = self.state.config.cache_dir();
7453 let context = format!("round {round}");
7454 let wait = self.repository_wait();
7455 let (e2e, verify_retried) = with_cache_lease(
7456 &mut self.state,
7457 cache_dir.as_deref(),
7458 "e2e",
7459 "e2e",
7460 &winner.worktree,
7461 &head,
7462 verify_timeout,
7463 &context,
7464 &wait,
7465 |state, budget| {
7466 let shell = shell.clone();
7467 let e2e_commands = e2e_commands.clone();
7468 let worktree = winner.worktree.clone();
7469 let context = context.clone();
7470 async move {
7471 run_e2e_with_retry(
7472 state,
7473 &shell,
7474 &e2e_commands,
7475 &worktree,
7476 budget,
7477 &context,
7478 )
7479 .await
7480 }
7481 },
7482 )
7483 .await?;
7484 (e2e, verify_retried, false, None)
7485 };
7486
7487 let expected = records.len();
7488 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7489 let incomplete = answered < expected;
7490 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7491 let policy = self.state.config.graph.incomplete_review;
7492 let clean = round_is_clean(
7493 blocking,
7494 e2e_ok,
7495 answered,
7496 expected,
7497 round_quota_missing,
7498 policy,
7499 );
7500
7501 let mut round_record = ReviewRound {
7502 round,
7503 head: head.clone(),
7504 verified_head: None,
7505 verified_at: None,
7506 reviews: records,
7507 e2e,
7508 verify_retried,
7509 e2e_deferred,
7510 e2e_defer_reason,
7511 fix: None,
7512 blocking,
7513 answered,
7514 expected,
7515 clean,
7516 progressed: false,
7517 vote_split,
7518 reconsideration,
7519 verdict: None,
7520 };
7521 round_record.verdict = ReviewVote::worst(
7525 round_record
7526 .final_votes()
7527 .into_iter()
7528 .map(|(_, _, vote)| vote),
7529 );
7530 if !matches!(
7541 round_record.e2e_status(),
7542 E2eStatus::Deferred | E2eStatus::NotConfigured
7543 ) {
7544 round_record.verified_head = Some(head.clone());
7545 round_record.verified_at = Some(Timestamp::now());
7546 }
7547
7548 if incomplete {
7549 let missing: Vec<String> = round_record
7550 .reviews
7551 .iter()
7552 .filter(|r| r.failed.is_some())
7553 .map(|r| format!("review-{}", r.reviewer))
7554 .collect();
7555 self.state.event(
7556 "review",
7557 format!(
7558 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7559 missing.join(", ")
7560 ),
7561 );
7562 }
7563
7564 if clean {
7565 self.state.event(
7566 "review",
7567 if incomplete && policy == IncompleteReviewPolicy::Warn {
7568 format!(
7569 "round {round}: clean (warn policy, incomplete panel) — no \
7570 blocking findings from the seats that answered, verification green"
7571 )
7572 } else if incomplete {
7573 format!(
7574 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7575 quorum) — no blocking findings from the seats that answered, \
7576 verification green",
7577 expected - answered
7578 )
7579 } else {
7580 format!("round {round}: clean — no blocking findings, verification green")
7581 },
7582 );
7583 self.state.reviews.push(round_record);
7584 self.state.status = RunStatus::Gating;
7585 self.state.save()?;
7586 return Ok(());
7587 }
7588
7589 if incomplete && blocking == 0 && e2e_ok {
7597 self.state.reviews.push(round_record);
7598 self.state.save()?;
7599 if round == max_rounds {
7600 self.state.status = RunStatus::Blocked;
7601 self.state.event(
7602 "review",
7603 format!(
7604 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7605 refusing to call it clean",
7606 expected - answered
7607 ),
7608 );
7609 return Ok(());
7610 }
7611 continue;
7612 }
7613
7614 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7626 self.state.reviews.push(round_record);
7627 return self
7628 .stop_unresolved_review(
7629 "the round's own verification could not run",
7630 &shell,
7631 &winner.worktree,
7632 )
7633 .await;
7634 }
7635
7636 if round == max_rounds {
7637 self.state.reviews.push(round_record);
7638 return self
7639 .stop_reviewing(
7640 &format!(
7641 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7642 ),
7643 &shell,
7644 &winner.worktree,
7645 )
7646 .await;
7647 }
7648
7649 if !self.fix_review_round(&winner, round_record).await? {
7650 return Ok(());
7651 }
7652 }
7653 Ok(())
7654 }
7655
7656 async fn fix_review_round(
7659 &mut self,
7660 winner: &Candidate,
7661 mut round_record: ReviewRound,
7662 ) -> Result<bool> {
7663 let round = round_record.round;
7664 let shell = self.state.config.shell();
7665 let step = self.run_fix_round(winner, &mut round_record).await?;
7666 if matches!(step, FixStep::Resource) {
7667 self.stop_for_resource()?;
7668 return Ok(false);
7669 }
7670 if self.state.reviews.len() < round {
7671 self.state.reviews.push(round_record);
7672 } else {
7673 self.state.reviews[round - 1] = round_record;
7674 }
7675 self.state.save()?;
7676
7677 if matches!(step, FixStep::Recorded { report_lost: true }) {
7690 self.stop_reviewing(
7691 "the fixer's adoption report never came back, even after resuming its \
7692 own seat; refusing to start another round against the same worktree \
7693 while that is unresolved",
7694 &shell,
7695 &winner.worktree,
7696 )
7697 .await?;
7698 return Ok(false);
7699 }
7700
7701 let streak = self
7702 .state
7703 .reviews
7704 .iter()
7705 .rev()
7706 .take_while(|r| !r.progressed)
7707 .count();
7708 if streak >= STAGNANT_LIMIT {
7709 self.stop_reviewing(
7710 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7711 &shell,
7712 &winner.worktree,
7713 )
7714 .await?;
7715 return Ok(false);
7716 }
7717 Ok(true)
7718 }
7719
7720 async fn run_fix_round(
7722 &mut self,
7723 winner: &Candidate,
7724 round_record: &mut ReviewRound,
7725 ) -> Result<FixStep> {
7726 let round = round_record.round;
7727 let max_rounds = self.state.config.graph.review_rounds;
7728 let run_id = self.state.id.clone();
7729 let prompts = self.state.config.prompts.clone();
7730 let language = self.state.config.graph.language.clone();
7731 let sessions = self.state.config.graph.sessions;
7732 let artifacts = self.merge_repair_artifacts();
7733 let base = self.landing_base();
7734 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7735 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7736 let this_round_verification = round_record.verification_summary(&head);
7737 let all_findings: Vec<_> = round_record
7738 .reviews
7739 .iter()
7740 .flat_map(|r| r.findings.iter().cloned())
7741 .collect();
7742 let blocking_findings: Vec<_> = all_findings
7745 .iter()
7746 .filter(|f| f.severity.blocks())
7747 .cloned()
7748 .collect();
7749 let fix_prompt = prompt::fix_in(
7750 &self.state.instruction,
7751 &blocking_findings,
7752 this_round_verification.as_ref(),
7753 round,
7754 max_rounds,
7755 &language,
7756 &self.state.config.graph.github_language,
7757 );
7758 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7759 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7760 let (job, seat, out) = self
7761 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7762 prompt: fix_prompt.clone(),
7763 spec,
7764 seat,
7765 cwd: winner.worktree.clone(),
7766 timeout,
7767 allow_write: true,
7768 sessions,
7769 artifacts: artifacts.clone(),
7770 stem: format!("fix-{round}"),
7771 handover: None,
7772 })
7773 .await;
7774 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7777 return Ok(FixStep::Resource);
7778 }
7779 let agent_id = seat.agent.clone();
7780
7781 let mut fix = FixRecord {
7782 agent: agent_id,
7783 addressed: Vec::new(),
7784 rejected: Vec::new(),
7785 notes: String::new(),
7786 committed: false,
7787 failed: None,
7788 duration_ms: 0,
7789 continuation: None,
7790 };
7791 let mut continuation = ContinuationRecord::not_needed();
7792 let mut final_seat = seat.clone();
7793 match out {
7794 AgentOutcome::Ok(o) => {
7795 fix.duration_ms = o.duration_ms;
7796 let parsed = verdict::extract_json::<FixReport>(&o.text);
7797 let incomplete_reason = match &parsed {
7804 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7805 "the reply parsed, but it reported a command whose own CLI never \
7806 confirmed an exit status"
7807 .to_owned(),
7808 ),
7809 Ok(_) => None,
7810 Err(e) => Some(e.to_string()),
7811 };
7812 match incomplete_reason {
7813 None => {
7814 let report = parsed.expect("checked Ok above");
7815 fix.addressed = report.addressed;
7816 fix.rejected = report.rejected;
7817 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7818 }
7819 Some(reason) => {
7820 let (resumed_seat, resolved, failure, cont) = self
7821 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7822 .await;
7823 fix.duration_ms += cont.cumulative_wait_ms;
7824 continuation = cont;
7825 final_seat = resumed_seat;
7826 match resolved {
7827 Some(report) => {
7828 fix.addressed = report.addressed;
7829 fix.rejected = report.rejected;
7830 fix.notes =
7831 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7832 }
7833 None => fix.failed = failure,
7834 }
7835 }
7836 }
7837 }
7838 AgentOutcome::Dropped(o) => {
7840 fix.duration_ms = o.duration_ms;
7841 let why = o
7842 .dropped
7843 .as_ref()
7844 .map(|d| d.why.as_str())
7845 .unwrap_or("the CLI ended the stream without delivering its answer");
7846 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7847 }
7848 AgentOutcome::Quota(o) => {
7849 self.state.quota.push(QuotaLoss::from_report(
7850 final_seat.key.clone(),
7851 "fix",
7852 o.quota.as_ref(),
7853 ));
7854 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7855 }
7856 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7857 }
7858 if self.state.resource_block.is_some() {
7859 self.state.seats.insert(final_seat.key.clone(), final_seat);
7860 return Ok(FixStep::Resource);
7861 }
7862 fix.continuation = Some(continuation);
7863 self.state.seats.insert(final_seat.key.clone(), final_seat);
7864 let coordination = self.coordinate(&winner.worktree).await?;
7865 if let Ok(r) = git::rescue_commit(
7866 &winner.worktree,
7867 &format!("magi: review round {round} fixes (uncommitted work)"),
7868 )
7869 .await
7870 {
7871 self.state.note_withheld("fix", &r.withheld);
7872 }
7873 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7874 fix.committed = after != before;
7875 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7883 drop(coordination);
7884 let progressed = diff_after != patch;
7885 let commit_note = if fix.committed {
7886 "committed"
7887 } else {
7888 "NO new commit"
7889 };
7890 let tree_note = if progressed {
7891 "changed vs base"
7892 } else {
7893 "unchanged vs base"
7894 };
7895 self.state.event(
7896 "fix",
7897 match &fix.failed {
7898 Some(reason) => {
7904 format!(
7905 "round {round}: fixer's adoption report was lost ({reason}); \
7906 {commit_note}, tree {tree_note}"
7907 )
7908 }
7909 None => format!(
7910 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7911 {tree_note}{}",
7912 fix.addressed.len(),
7913 fix.rejected.len(),
7914 if continuation.outcome == ContinuationOutcome::Resumed {
7915 format!(
7916 " (adoption report recovered after {} continuation(s))",
7917 continuation.attempts
7918 )
7919 } else {
7920 String::new()
7921 },
7922 ),
7923 },
7924 );
7925 round_record.fix = Some(fix);
7926 round_record.progressed = progressed;
7927 Ok(FixStep::Recorded {
7928 report_lost: matches!(
7929 continuation.outcome,
7930 ContinuationOutcome::Exhausted
7931 | ContinuationOutcome::QuotaLost
7932 | ContinuationOutcome::NoSession
7933 ),
7934 })
7935 }
7936
7937 fn record_contested_handoff(&mut self) {
7971 if self.state.contested_handoff.is_some() {
7972 return;
7973 }
7974 let Some(contested) = self
7975 .state
7976 .reviews
7977 .last()
7978 .and_then(ReviewRound::contested_handoff)
7979 else {
7980 return;
7981 };
7982 self.state.event(
7983 "review",
7984 format!(
7985 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7986 wait for the owner's approval",
7987 contested.findings.len(),
7988 contested.rejecters.len()
7989 ),
7990 );
7991 self.state.contested_handoff = Some(contested);
7992 }
7993
7994 async fn stop_unresolved_review(
7995 &mut self,
7996 why: &str,
7997 shell: &[String],
7998 worktree: &Path,
7999 ) -> Result<()> {
8000 self.stop_reviewing_inner(why, shell, worktree, true).await
8001 }
8002
8003 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8004 self.stop_reviewing_inner(why, shell, worktree, false).await
8005 }
8006
8007 async fn stop_reviewing_inner(
8008 &mut self,
8009 why: &str,
8010 shell: &[String],
8011 worktree: &Path,
8012 repairable: bool,
8013 ) -> Result<()> {
8014 let round_idx = self.state.reviews.len() - 1;
8015 let needs_catchup_run = matches!(
8023 self.state.reviews[round_idx].e2e_status(),
8024 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8025 );
8026 if needs_catchup_run {
8027 let round = self.state.reviews[round_idx].round;
8028 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8029 let commands = self.state.config.verify.e2e.clone();
8030 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8031 let cache_dir = self.state.config.cache_dir();
8032 let context = format!(
8033 "round {round}: verification unresolved, catching up before the final decision"
8034 );
8035 let wait = self.repository_wait();
8036 let (outcomes, verify_retried) = with_cache_lease(
8037 &mut self.state,
8038 cache_dir.as_deref(),
8039 "e2e",
8040 "e2e",
8041 worktree,
8042 &attempted_head,
8043 timeout,
8044 &context,
8045 &wait,
8046 |state, budget| {
8047 let shell = shell.to_vec();
8048 let commands = commands.clone();
8049 let context = context.clone();
8050 async move {
8051 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8052 .await
8053 }
8054 },
8055 )
8056 .await?;
8057 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8058 self.state.record_base_progress(&attempted_head);
8059 }
8060 let last = &mut self.state.reviews[round_idx];
8061 last.e2e = outcomes;
8062 last.verify_retried = verify_retried;
8063 last.verified_head = Some(attempted_head);
8070 last.verified_at = Some(Timestamp::now());
8071 if verify_inconclusive(&last.e2e) {
8072 self.state.save()?;
8079 return Ok(());
8080 }
8081 last.e2e_deferred = false;
8082 if last.answered == last.expected {
8087 last.clean = round_is_clean(
8088 last.blocking,
8089 last.e2e.iter().all(CommandOutcome::ok),
8090 last.answered,
8091 last.expected,
8092 0,
8093 self.state.config.graph.incomplete_review,
8094 );
8095 }
8096 }
8097 let last = &self.state.reviews[round_idx];
8098 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8099
8100 match last.e2e_status() {
8101 E2eStatus::Failed => {
8102 let red: Vec<String> = last
8103 .e2e
8104 .iter()
8105 .filter(|o| !o.ok())
8106 .map(|o| {
8107 format!(
8108 "`{}` -> {:?}\n{}",
8109 o.command,
8110 o.code,
8111 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8112 )
8113 })
8114 .collect();
8115 let can_repair = last.blocking == 0 && last.fix.is_none();
8116 self.state.reviews[round_idx].clean = false;
8117 self.state
8118 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8119 self.state.status = RunStatus::Blocked;
8120 if repairable
8121 && can_repair
8122 && self.state.reviews.len() < self.state.config.graph.review_rounds
8123 {
8124 self.state.status = RunStatus::Reviewing;
8125 }
8126 }
8127 E2eStatus::ResourceBlocked => {
8132 self.state.event(
8133 "review",
8134 format!(
8135 "{why}; e2e could not run (resource unavailable); not \
8136 deciding yet"
8137 ),
8138 );
8139 }
8140 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8141 self.state.event(
8142 "review",
8143 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8144 );
8145 self.record_contested_handoff();
8146 self.state.status = RunStatus::Gating;
8147 }
8148 }
8149 self.state.save()?;
8150 Ok(())
8151 }
8152
8153 async fn gate(&mut self) -> Result<()> {
8156 if self.hold_base_sync().await? {
8157 return Ok(());
8158 }
8159 if self.state.status == RunStatus::Failed
8171 || self
8172 .state
8173 .base_sync
8174 .as_ref()
8175 .is_some_and(|s| s.conflict.is_some())
8176 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8177 != Some(RunStatus::Gating)
8178 {
8179 return Ok(());
8180 }
8181 let Some(winner) = self.state.winner().cloned() else {
8182 return Ok(());
8183 };
8184 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8185 self.state.reconcile_verification_head(&head);
8186 self.state.save()?;
8187 let wait = self.repository_wait();
8188 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8189 return Ok(());
8190 }
8191 if self.state.gate_ran {
8192 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8203 self.state.status = RunStatus::Blocked;
8204 self.state.save()?;
8205 }
8206 return Ok(());
8207 }
8208 self.state.status = RunStatus::Gating;
8209 let mut outcomes = self.run_gate(&winner).await?;
8210 let mut attempted_head = self.state.gate_head.clone();
8211 loop {
8212 if verify_inconclusive(&outcomes) {
8223 self.state.save()?;
8224 return Ok(());
8225 }
8226 if outcomes.iter().all(CommandOutcome::ok) {
8227 break;
8228 }
8229 let fixes_before = self.state.gate_fixes.len();
8230 match self.gate_fix_round(&winner, &outcomes).await? {
8231 GateFix::Retry => {
8232 outcomes = self.run_gate(&winner).await?;
8233 attempted_head = self.state.gate_head.clone();
8234 }
8235 GateFix::Stop => {
8236 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8237 if self.state.gate_fixes.len() > fixes_before
8238 && attempted_head.as_deref() != Some(current.as_str())
8239 && verify_current_head(&mut self.state, &winner, &wait).await?
8240 {
8241 outcomes = self.run_gate(&winner).await?;
8244 attempted_head = self.state.gate_head.clone();
8245 } else {
8246 break;
8247 }
8248 }
8249 GateFix::Defer => {
8250 self.state.save()?;
8251 return Ok(());
8252 }
8253 }
8254 }
8255 let passed = outcomes.iter().all(CommandOutcome::ok);
8256 self.state.gate = outcomes;
8257 self.state.gate_ran = true;
8258 self.state.gate_head = attempted_head;
8259 if !self.state.gate.is_empty()
8260 && !verify_inconclusive(&self.state.gate)
8261 && let Some(head) = self.state.gate_head.clone()
8262 {
8263 self.state.record_base_progress(&head);
8264 }
8265 self.state.save()?;
8266 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8267 return Ok(());
8268 }
8269 if !passed {
8270 self.state.status = RunStatus::Blocked;
8271 let spent = self.state.gate_fixes.len();
8272 self.state.event(
8273 "gate",
8274 if spent == 0 {
8275 "gate failed; not merging".to_owned()
8276 } else {
8277 format!("gate failed after {spent} gate-fix round(s); not merging")
8278 },
8279 );
8280 }
8281 self.state.save()?;
8282 Ok(())
8283 }
8284
8285 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8295 let commands = state.config.verify.pre_gate.clone();
8296 if commands.is_empty() {
8297 return;
8298 }
8299 let shell = state.config.shell();
8300 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8301 let (outcomes, _) = run_commands(
8302 state,
8303 "pre_gate",
8304 "pre_gate",
8305 0,
8306 &shell,
8307 &commands,
8308 &winner.worktree,
8309 timeout,
8310 )
8311 .await;
8312 for o in &outcomes {
8313 if !o.ok() {
8314 tracing::warn!(
8315 "pre_gate `{}` failed ({:?}); the gate decides",
8316 o.command,
8317 o.code
8318 );
8319 }
8320 state.event(
8321 "pre_gate",
8322 format!(
8323 "`{}` -> {}",
8324 o.command,
8325 if o.ok() {
8326 "pass".to_owned()
8327 } else {
8328 format!(
8329 "FAIL ({:?})\n{}",
8330 o.code,
8331 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8332 )
8333 }
8334 ),
8335 );
8336 }
8337 state.pre_gate = outcomes;
8338 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8339 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8340 Ok(head) => {
8341 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8342 state.pre_gate_commit = Some(head);
8343 }
8344 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8345 },
8346 Ok(false) => {}
8347 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8348 }
8349 if let Err(e) = state.save() {
8350 tracing::warn!("could not persist the pre_gate record: {e:#}");
8351 }
8352 }
8353
8354 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8357 let wait = self.repository_wait();
8358 run_landing_gate(&mut self.state, winner, &wait).await
8359 }
8360
8361 async fn gate_fix_round(
8374 &mut self,
8375 winner: &Candidate,
8376 outcomes: &[CommandOutcome],
8377 ) -> Result<GateFix> {
8378 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8379 g.phase == crate::repair_budget::Phase::Fixing
8380 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8381 }) {
8382 self.state.gate_fixes.len().saturating_add(1)
8383 } else {
8384 self.state.config.graph.gate_fix_rounds
8385 };
8386 let spent = self.state.gate_fixes.len();
8387 if spent >= cap {
8388 if cap > 0 {
8389 self.state.event(
8390 "gate",
8391 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8392 );
8393 }
8394 return Ok(GateFix::Stop);
8395 }
8396 if !gate_fixable(outcomes) {
8397 self.state.event(
8398 "gate",
8399 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8400 command or similar); not spending a fix round on it",
8401 );
8402 return Ok(GateFix::Stop);
8403 }
8404 let min_free = self.state.config.disk.min_free_bytes;
8405 if min_free > 0 {
8406 match crate::disk::free_bytes(&winner.worktree) {
8407 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8408 Ok(free) => {
8409 self.state.event(
8410 "gate",
8411 format!(
8412 "only {free} bytes free ({min_free} required by `[disk] \
8413 min_free_bytes`); not spending a fix round on a failure the disk \
8414 may explain"
8415 ),
8416 );
8417 return Ok(GateFix::Stop);
8418 }
8419 Err(e) => {
8420 self.state.event(
8421 "gate",
8422 format!("free disk space could not be measured ({e:#}); no fix round"),
8423 );
8424 return Ok(GateFix::Stop);
8425 }
8426 }
8427 }
8428
8429 let attempt = spent + 1;
8430 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8431 let base = self.landing_base();
8432 let fix_prompt = prompt::gate_fix_in(
8433 &self.state.instruction,
8434 &failed,
8435 attempt,
8436 cap,
8437 &self.state.config.graph.language,
8438 &self.state.config.graph.github_language,
8439 );
8440 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8441 let sessions = self.state.config.graph.sessions;
8442 let artifacts = self.merge_repair_artifacts();
8443 self.state.event(
8444 "gate",
8445 format!("gate failed; gate-fix round {attempt} of {cap}"),
8446 );
8447 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8448 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8449 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8452 head: before.clone(),
8453 passed: false,
8454 outcomes: Vec::new(),
8455 });
8456 self.state.save()?;
8457 let (_, seat, out) = self
8458 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8459 prompt: fix_prompt.clone(),
8460 spec,
8461 seat,
8462 cwd: winner.worktree.clone(),
8463 timeout,
8464 allow_write: true,
8465 sessions,
8466 artifacts: artifacts.clone(),
8467 stem: format!("gate-fix-{attempt}"),
8468 handover: None,
8469 })
8470 .await;
8471 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8474 return Ok(GateFix::Defer);
8475 }
8476 let mut record = GateFixRecord {
8477 agent: seat.agent.clone(),
8478 failed,
8479 notes: String::new(),
8480 committed: false,
8481 error: None,
8482 };
8483 match out {
8484 AgentOutcome::Ok(o) => {
8485 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8488 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8489 }
8490 }
8491 AgentOutcome::Dropped(_) => {
8492 record.error = Some("the CLI dropped the stream".to_owned());
8493 }
8494 AgentOutcome::Quota(o) => {
8495 self.state.quota.push(QuotaLoss::from_report(
8496 seat.key.clone(),
8497 "gate-fix",
8498 o.quota.as_ref(),
8499 ));
8500 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8501 }
8502 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8503 }
8504 self.state.seats.insert(seat.key.clone(), seat);
8505 let coordination = self.coordinate(&winner.worktree).await?;
8506 if let Ok(r) = git::rescue_commit(
8507 &winner.worktree,
8508 &format!("magi: gate fix {attempt} (uncommitted work)"),
8509 )
8510 .await
8511 {
8512 self.state.note_withheld("gate-fix", &r.withheld);
8513 }
8514 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8515 record.committed = after != before;
8516 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8517 let note = record.error.clone();
8518 self.state.gate_fixes.push(record);
8519 self.state.save()?;
8520 if !changed {
8521 self.state.event(
8522 "gate",
8523 match note {
8524 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8525 None => format!("gate-fix round {attempt}: the tree did not change"),
8526 },
8527 );
8528 return Ok(GateFix::Stop);
8529 }
8530 self.state.event(
8531 "gate",
8532 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8533 );
8534
8535 drop(coordination);
8536 let wait = self.repository_wait();
8537 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8538 }
8539
8540 async fn guarded_pr_message(
8547 &mut self,
8548 winner: &Candidate,
8549 facts: Option<&BranchFacts>,
8550 posting: bool,
8551 ) -> Result<Option<PrMessage>> {
8552 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8553 let head = match final_head.as_deref() {
8554 Some(h) => crate::pr_outcome::FinalHead::At(h),
8555 None => crate::pr_outcome::FinalHead::Unknown,
8556 };
8557 let summarized = if posting {
8559 Some(self.summarize_change(winner, final_head.as_deref()).await)
8560 } else {
8561 None
8562 };
8563 if self.state.resource_block.is_some() {
8568 self.state.event(
8569 "pr-summary",
8570 "a shared resource was busy; nothing was published, resume to summarize again",
8571 );
8572 return Ok(None);
8573 }
8574 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8575 anyhow::bail!(
8576 "the final commit moved during public description generation; retry before publishing"
8577 );
8578 }
8579 let note = match (&summarized, &self.state.change_summary) {
8580 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8581 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8582 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8583 (None, _) => ChangeNote::NotAsked,
8584 };
8585 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8586 if !posting {
8587 let identity = crate::scrub::Identity::current();
8588 return Ok(Some(PrMessage {
8589 title: crate::scrub::scrub(&pr.title, &identity),
8590 body: crate::scrub::scrub(&pr.body, &identity),
8591 }));
8592 }
8593 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8596 if let Some(known) = self
8597 .state
8598 .github_text
8599 .clone()
8600 .filter(|g| g.fingerprint == raw_fp)
8601 {
8602 return self.settle_withheld(winner, &pr, known).await;
8603 }
8604 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8605 let mut violations =
8606 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8607 let identity = crate::scrub::Identity::current();
8608 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8609 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8610 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8611 {
8612 violations.push(crate::github_text::Violation::SensitiveData);
8613 }
8614 let has_language = violations
8618 .iter()
8619 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8620 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8621 self.state.event(
8622 "github-text",
8623 if has_language {
8624 format!("description rejected: {violations:?}; requesting one rewrite")
8625 } else {
8626 format!("description rejected: {violations:?}; asking the owner")
8627 },
8628 );
8629 let mut rewritten = false;
8630 let mut rejected_rewrite = None;
8631 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8632 {
8633 let output = text;
8634 if let Some(title) = summary_title(&output) {
8635 let raw_body = summary_without_title(&output);
8638 let clean_title = crate::scrub::scrub(&title, &identity);
8639 let mut body = crate::scrub::scrub(&raw_body, &identity);
8640 let redacted = clean_title != title || body != raw_body;
8641 let title = clean_title;
8642 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8643 if !complete {
8644 self.state.event(
8645 "github-text",
8646 "description rewrite omitted required findings",
8647 );
8648 }
8649 if !body.trim().is_empty() {
8650 body.push_str("\n\n");
8651 body.push_str(&public_pr_evidence(&self.state, head));
8652 }
8653 let marker = format!("magi:run/{}", self.state.id);
8654 if !body.trim().is_empty() && !body.contains(&marker) {
8655 body.push_str(&format!("\n\n{marker}"));
8656 }
8657 let rewrite = PrMessage {
8658 title: title.clone(),
8659 body: body.clone(),
8660 };
8661 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8662 let mut failed = crate::github_text::check_for(
8663 &self.state.config.graph,
8664 &title,
8665 &body,
8666 redecision,
8667 );
8668 if (!crate::github_text::shareable(&title)
8669 || !crate::github_text::shareable(&body))
8670 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8671 {
8672 failed.push(crate::github_text::Violation::SensitiveData);
8673 }
8674 if complete
8675 && !title.trim().is_empty()
8676 && !body.trim().is_empty()
8677 && !failed.is_empty()
8678 {
8679 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8680 title: title.clone(),
8681 body: body.clone(),
8682 failed: failed
8683 .iter()
8684 .map(|v| crate::github_text::category(*v).to_owned())
8685 .collect(),
8686 });
8687 }
8688 if complete
8689 && !title.trim().is_empty()
8690 && !raw_body.trim().is_empty()
8691 && failed.is_empty()
8692 {
8693 decision = redecision;
8694 pr = PrMessage { title, body };
8695 rewritten = true;
8696 self.state
8697 .event("github-text", "description rewrite passed");
8698 if redacted {
8699 self.state.event(
8700 "github-text",
8701 "sensitive data removed from description rewrite",
8702 );
8703 }
8704 }
8705 }
8706 }
8707 if !rewritten {
8708 self.state.event(
8709 "github-text",
8710 "description rewrite unavailable or rejected; asking the owner",
8711 );
8712 if let Some(w) =
8713 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8714 {
8715 self.retire_github_text_question("the withheld text changed");
8718 let record = crate::run::GithubTextAsk {
8719 rewrite: rejected_rewrite,
8720 fingerprint: raw_fp,
8721 title: w.title,
8722 body: w.body,
8723 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8724 question: None,
8725 asks: 0,
8726 resolved: false,
8727 chosen_title: None,
8728 choice: None,
8729 };
8730 return self.settle_withheld(winner, &pr, record).await;
8731 }
8732 }
8733 }
8734 let (title, body) =
8735 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8736 Ok(Some(PrMessage { title, body }))
8737 }
8738
8739 async fn summarize_change(
8750 &mut self,
8751 winner: &Candidate,
8752 head: Option<&str>,
8753 ) -> std::result::Result<(), String> {
8754 let Some(head) = head else {
8755 return Err("the final commit could not be read".to_owned());
8756 };
8757 let remote = self.state.config.merge.remote.clone();
8758 let tracking = format!("{remote}/{}", self.state.base_branch);
8759 if !git::rev_exists(&winner.worktree, &tracking).await {
8760 return Err(format!("the remote base `{tracking}` could not be read"));
8761 }
8762 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8763 Ok(b) if !b.is_empty() => b,
8764 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8765 };
8766 let evidence = crate::pr_outcome::summary_evidence(
8767 &self.state,
8768 crate::pr_outcome::FinalHead::At(head),
8769 );
8770 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8771 if self
8772 .state
8773 .change_summary
8774 .as_ref()
8775 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8776 {
8777 return Ok(());
8778 }
8779 if self.state.change_summary.take().is_some() {
8780 self.state.event(
8781 "pr-summary",
8782 "the final diff or evidence changed; the earlier change summary was discarded",
8783 );
8784 }
8785 if let Err(e) = write_artifact(
8787 &self.state,
8788 &format!("public-description-{fingerprint}.md"),
8789 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8790 ) {
8791 tracing::warn!("could not save public description evidence: {e:#}");
8792 }
8793 let chain = match agent::pick_chain(
8794 &self.state.config.agents,
8795 self.state.config.roles.synthesizer.as_ref(),
8796 &agent::installed,
8797 "synthesizer",
8798 ) {
8799 Ok(chain) => chain,
8800 Err(e) => {
8801 tracing::warn!("no agent for the change summary: {e:#}");
8802 self.state.event("pr-summary", "no summary agent available");
8803 return Err("no agent was available to summarize it".to_owned());
8804 }
8805 };
8806 let subjects = git::subjects(&winner.worktree, &base, head)
8807 .await
8808 .unwrap_or_default();
8809 let stat = git::diff_stat(&winner.worktree, &base, head)
8810 .await
8811 .unwrap_or_default();
8812 let stat: String = stat.chars().take(6_000).collect();
8813 let mut prompt = crate::prompt::change_summary_in(
8814 &base,
8815 head,
8816 &subjects,
8817 stat.trim(),
8818 &self.state.config.graph.github_language,
8819 );
8820 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8821 let sessions = self.state.config.graph.sessions;
8822 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8823 let prompts = self.state.config.prompts.clone();
8824 let cache = self.state.config.cache_dir();
8825 let run = self.state.id.clone();
8826 for (i, spec) in chain.into_iter().enumerate() {
8827 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8828 let github_spec = match agent::for_github_text(
8829 &spec,
8830 &self.state.config.graph.github_language,
8831 &winner.worktree,
8832 ) {
8833 Ok(spec) => spec,
8834 Err(e) => {
8835 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8836 continue;
8837 }
8838 };
8839 let job = SeatJob {
8840 spec: github_spec,
8841 seat,
8842 cwd: winner.worktree.clone(),
8843 prompt: prompt.clone(),
8844 timeout,
8845 allow_write: false,
8846 sessions,
8847 artifacts: agent::artifacts_dir(&self.state.dir()),
8848 stem: if i == 0 {
8849 "change-summary".to_owned()
8850 } else {
8851 format!("change-summary-{}", spec.id)
8852 },
8853 handover: None,
8854 };
8855 let ctx = WaveCtx {
8856 run: &run,
8857 node: "pr-summary",
8858 prompts: &prompts,
8859 cache: cache.as_deref(),
8860 round: None,
8861 carry_seats: false,
8862 pauses: Some((&self.pause, &self.interrupt)),
8863 };
8864 let (seat, outcome) =
8865 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8866 self.state.seats.insert(seat.key.clone(), seat);
8867 let text = match outcome {
8868 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8869 AgentOutcome::ResourceBlocked(why) => {
8870 if self.state.resource_block.is_none() {
8873 self.state.resource_block = Some(crate::run::ResourceBlock {
8874 cause: crate::run::ResourceBlockCause::Other,
8875 node: "pr-summary".to_owned(),
8876 kind: "shared resource".to_owned(),
8877 detail: why,
8878 at: Timestamp::now(),
8879 });
8880 }
8881 return Err("a shared resource was busy".to_owned());
8882 }
8883 _ => continue,
8884 };
8885 let blind = &self.state.config.blind;
8886 let Some(title) = summary_title(&text) else {
8887 continue;
8888 };
8889 let body: String = summary_without_title(&text)
8890 .lines()
8891 .filter(|l| !l.contains("magi:run/"))
8892 .collect::<Vec<_>>()
8893 .join("\n");
8894 let body = blind::sanitize_prose(body.trim(), blind);
8895 let title = blind::sanitize_prose(&title, blind);
8896 if body.trim().is_empty() || title.trim().is_empty() {
8897 continue;
8898 }
8899 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8900 continue;
8901 }
8902 if git::rev_parse(&winner.worktree, "HEAD")
8904 .await
8905 .ok()
8906 .as_deref()
8907 != Some(head)
8908 {
8909 return Err("the branch moved while the summary was written".to_owned());
8910 }
8911 if git::merge_base(&winner.worktree, &tracking, head)
8912 .await
8913 .ok()
8914 .as_deref()
8915 != Some(base.as_str())
8916 || crate::pr_outcome::summary_evidence(
8917 &self.state,
8918 crate::pr_outcome::FinalHead::At(head),
8919 ) != evidence
8920 {
8921 return Err("the summary evidence changed while it was written".to_owned());
8922 }
8923 self.state.change_summary = Some(crate::run::ChangeSummary {
8924 head: head.to_owned(),
8925 base,
8926 evidence: fingerprint,
8927 title,
8928 body,
8929 });
8930 self.state.event(
8931 "pr-summary",
8932 format!("change summary written by {} for the final diff", spec.id),
8933 );
8934 if let Err(e) = self.state.save() {
8935 tracing::warn!("could not save the change summary: {e:#}");
8936 }
8937 return Ok(());
8938 }
8939 self.state
8940 .event("pr-summary", "no agent produced a change summary");
8941 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8942 }
8943
8944 async fn rewrite_pr_text(
8954 &mut self,
8955 winner: &Candidate,
8956 pr: &PrMessage,
8957 violations: &[crate::github_text::Violation],
8958 ) -> Option<String> {
8959 let prompt = format!(
8960 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8961 pr.title, pr.body
8962 );
8963 let sessions = self.state.config.graph.sessions;
8964 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8965 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8968 let key = format!("impl-{}", winner.label);
8969 if let Some(spec) = self
8970 .state
8971 .config
8972 .agents
8973 .iter()
8974 .find(|a| a.id == winner.agent)
8975 .cloned()
8976 && let Some(existing) = self.state.seats.get(&key)
8977 && existing.agent == spec.id
8978 && agent::has_session(spec.kind, existing, sessions)
8979 {
8980 attempts.push((spec, Some(existing.clone())));
8981 }
8982 match agent::pick_chain(
8983 &self.state.config.agents,
8984 self.state.config.roles.language_judge.as_ref(),
8985 &agent::installed,
8986 "language_judge",
8987 ) {
8988 Ok(chain) => {
8989 for spec in chain {
8990 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8991 continue;
8992 }
8993 attempts.push((spec, None));
8994 }
8995 }
8996 Err(e) if attempts.is_empty() => {
8997 tracing::warn!("no agent for the description rewrite: {e:#}");
8998 self.state
8999 .event("github-text", "no rewrite agent available");
9000 return None;
9001 }
9002 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9003 }
9004 let prompts = self.state.config.prompts.clone();
9005 let cache = self.state.config.cache_dir();
9006 let run = self.state.id.clone();
9007 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9008 let resumed = resumed_seat.is_some();
9009 let seat = resumed_seat.unwrap_or_else(|| {
9010 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9011 });
9012 let github_spec = match agent::for_github_text(
9013 &spec,
9014 &self.state.config.graph.github_language,
9015 &winner.worktree,
9016 ) {
9017 Ok(spec) => spec,
9018 Err(e) => {
9019 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9020 continue;
9021 }
9022 };
9023 let job = SeatJob {
9024 spec: github_spec,
9025 seat,
9026 cwd: winner.worktree.clone(),
9027 prompt: prompt.clone(),
9028 timeout: retry_budget(timeout_full, resumed),
9029 allow_write: false,
9030 sessions,
9031 artifacts: agent::artifacts_dir(&self.state.dir()),
9032 stem: if i == 0 {
9033 "github-text-rewrite".to_owned()
9034 } else {
9035 format!("github-text-rewrite-{}", spec.id)
9036 },
9037 handover: None,
9038 };
9039 let ctx = WaveCtx {
9040 run: &run,
9041 node: "github-text",
9042 prompts: &prompts,
9043 cache: cache.as_deref(),
9044 round: None,
9045 carry_seats: false,
9046 pauses: Some((&self.pause, &self.interrupt)),
9047 };
9048 let (seat, outcome) =
9049 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9050 self.state.seats.insert(seat.key.clone(), seat);
9051 match outcome {
9052 AgentOutcome::Ok(o)
9053 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9054 {
9055 return Some(o.text);
9056 }
9057 AgentOutcome::ResourceBlocked(_) => return None,
9058 _ => {}
9059 }
9060 }
9061 None
9062 }
9063
9064 fn retire_github_text_question(&mut self, why: &str) {
9066 let Some(id) = self
9067 .state
9068 .github_text
9069 .as_ref()
9070 .and_then(|g| g.question.clone())
9071 else {
9072 return;
9073 };
9074 let _ = ask::Questions::open().update(&id, |q| {
9075 q.abandon(why);
9076 Ok(())
9077 });
9078 }
9079
9080 async fn settle_withheld(
9084 &mut self,
9085 winner: &Candidate,
9086 pr: &PrMessage,
9087 mut g: crate::run::GithubTextAsk,
9088 ) -> Result<Option<PrMessage>> {
9089 use crate::github_text as gt;
9090 let store = ask::Questions::open();
9091 let timeout = self.state.config.graph.answer_timeout;
9092 if !g.resolved {
9093 let mut standing = g
9094 .question
9095 .as_ref()
9096 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9097 if let Some(q) = standing.as_ref()
9098 && q.status.open()
9099 && gt::expired(q, timeout)
9100 {
9101 let why = format!("no answer within {}s of asking", timeout.max(1));
9102 standing = store
9103 .update(&q.id, |q| {
9104 q.abandon(&why);
9105 Ok(())
9106 })
9107 .ok()
9108 .map(|(q, ())| q);
9109 }
9110 let mut ask_again: Option<Option<Vec<String>>> = None;
9111 match standing {
9112 None if g.asks <= 1 => {
9115 g.asks = 0;
9116 ask_again = Some(None);
9117 }
9118 None => {
9119 g.resolved = true;
9120 g.chosen_title = None;
9121 g.choice = Some(GithubTextChoice::Fallback);
9122 }
9123 Some(q) if q.status.open() => {
9124 self.park_on_github_text(&g)?;
9125 return Ok(None);
9126 }
9127 Some(q) => match gt::read_reply(&q) {
9128 gt::Reply::Fallback => {
9129 g.resolved = true;
9130 g.chosen_title = None;
9131 g.choice = Some(GithubTextChoice::Fallback);
9132 }
9133 gt::Reply::Original => {
9134 g.resolved = true;
9135 g.chosen_title = None;
9136 g.choice = Some(GithubTextChoice::Original);
9137 }
9138 gt::Reply::Rewrite => {
9139 g.resolved = true;
9140 g.chosen_title = None;
9141 g.choice = Some(if g.rewrite.is_some() {
9142 GithubTextChoice::Rewrite
9143 } else {
9144 GithubTextChoice::Fallback
9145 });
9146 }
9147 gt::Reply::Title(text) => {
9148 let first = text
9149 .lines()
9150 .map(str::trim)
9151 .find(|l| !l.is_empty())
9152 .unwrap_or("");
9153 let decision = if first.is_empty() {
9154 None
9155 } else {
9156 let candidate = PrMessage {
9157 title: first.to_owned(),
9158 body: String::new(),
9159 };
9160 self.judge_pr_language(&winner.worktree, &candidate).await
9161 };
9162 match gt::vet_title(
9163 &text,
9164 decision,
9165 self.state.config.graph.github_english(),
9166 ) {
9167 Ok(title) => {
9168 g.resolved = true;
9169 g.chosen_title = Some(title);
9170 g.choice = Some(GithubTextChoice::MyTitle);
9171 }
9172 Err(cats) if g.asks < 2 => {
9173 ask_again = Some(Some(
9174 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9175 ))
9176 }
9177 Err(_) => {
9178 g.resolved = true;
9179 g.chosen_title = None;
9180 g.choice = Some(GithubTextChoice::Fallback);
9181 }
9182 }
9183 }
9184 },
9185 }
9186 if let Some(rejected) = ask_again {
9187 let w = gt::Withheld {
9190 title: g.title,
9191 body: g.body,
9192 categories: g.categories.clone(),
9193 };
9194 let artifact = match crate::run::write_artifact(
9197 &self.state,
9198 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9199 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9200 ) {
9201 Ok(p) => Some(p),
9202 Err(e) => {
9203 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9204 None
9205 }
9206 };
9207 let (detail, comparison) = gt::question_content(
9208 &w,
9209 &pr.title,
9210 &pr.body,
9211 &self.state.id,
9212 rejected.as_deref(),
9213 artifact.as_deref(),
9214 g.rewrite.as_ref(),
9215 );
9216 let mut q = ask::Question::new(
9217 self.state.id.clone(),
9218 gt::ASK_NODE.to_owned(),
9219 gt::ASK_SEAT.to_owned(),
9220 gt::question_summary(&self.state.config.graph.language, &w),
9221 detail,
9222 gt::question_choices(&w, g.rewrite.as_ref()),
9223 );
9224 q.posting_comparison = Some(comparison);
9225 q.answer_timeout = timeout;
9226 g.question = Some(q.id.clone());
9229 g.asks += 1;
9230 self.state.github_text = Some(g.clone());
9231 self.state.event(
9232 "github-text",
9233 format!("asking the owner about the withheld text ({})", q.short()),
9234 );
9235 self.state.save()?;
9236 store
9237 .put_executable(&mut q)
9238 .context("file the posting-gate question")?;
9239 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9240 tracing::warn!(
9241 "could not notify about posting-gate question {}: {e:#}",
9242 q.short()
9243 );
9244 }
9245 self.park_on_github_text(&g)?;
9246 return Ok(None);
9247 }
9248 }
9249 let resolved = g.chosen_title.clone();
9250 let original = g.choice == Some(GithubTextChoice::Original);
9251 self.state.event(
9253 "github-text",
9254 if original {
9255 format!(
9256 "posting the original text for: {}",
9257 match (g.title, g.body) {
9258 (true, true) => "title, description",
9259 (true, false) => "title",
9260 _ => "description",
9261 }
9262 )
9263 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9264 "posting the saved rewrite title and description".to_owned()
9265 } else if resolved.is_some() {
9266 "posting the owner's replacement title".to_owned()
9267 } else {
9268 "posting the neutral text for the withheld fields".to_owned()
9269 },
9270 );
9271 self.state.github_text = Some(g.clone());
9272 self.state.save()?;
9273 if g.choice == Some(GithubTextChoice::Rewrite)
9274 && let Some(r) = g.rewrite.as_ref()
9275 {
9276 return Ok(Some(PrMessage {
9277 title: r.title.clone(),
9278 body: r.body.clone(),
9279 }));
9280 }
9281 let w = gt::Withheld {
9282 title: g.title,
9283 body: g.body,
9284 categories: g.categories.clone(),
9285 };
9286 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9287 if let Some(t) = resolved {
9288 title = t;
9289 }
9290 let english = gt::LanguageDecision {
9292 title_english: true,
9293 body_english: true,
9294 body_complete: true,
9295 };
9296 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9297 if original {
9298 if g.title {
9301 title = pr.title.clone();
9302 }
9303 if g.body {
9304 body = pr.body.clone();
9305 }
9306 }
9307 Ok(Some(PrMessage { title, body }))
9308 }
9309
9310 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9311 self.state.github_text = Some(g.clone());
9312 self.state.end_pass_parked(ParkCause::Question);
9313 self.state.parked = true;
9314 self.state.event(
9315 "github-text",
9316 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9317 );
9318 self.state.save()
9319 }
9320
9321 async fn judge_pr_language(
9324 &mut self,
9325 cwd: &Path,
9326 pr: &PrMessage,
9327 ) -> Option<crate::github_text::LanguageDecision> {
9328 self.state.config.roles.language_judge.as_ref()?;
9329 if !self.state.config.graph.language_gate() {
9330 return None;
9331 }
9332 let decision =
9333 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9334 self.state.event(
9335 "github-text",
9336 if decision.is_some() {
9337 "language decided by the language judge"
9338 } else {
9339 "language judge unavailable; using built-in heuristics"
9340 },
9341 );
9342 decision
9343 }
9344
9345 async fn merge(&mut self) -> Result<()> {
9346 if self.resource_halt()? || self.hold_base_sync().await? {
9347 return Ok(());
9348 }
9349 if self
9364 .state
9365 .base_sync
9366 .as_ref()
9367 .is_some_and(|s| s.conflict.is_some())
9368 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9369 != Some(RunStatus::Gating)
9370 || !self.state.gate_allows_progress()
9378 {
9379 return Ok(());
9380 }
9381 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9390 return Ok(());
9391 }
9392 let Some(winner) = self.state.winner().cloned() else {
9393 return Ok(());
9394 };
9395 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9396 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9397 self.state.status = RunStatus::Blocked;
9398 self.state.event(
9399 "gate",
9400 "winner changed after gate; fresh verification required",
9401 );
9402 self.state.save()?;
9403 return Ok(());
9404 }
9405 let wait = self.repository_wait();
9406 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9407 return Ok(());
9408 }
9409 let repo = self.state.repo.clone();
9410 let base = self.state.base_branch.clone();
9411 let mode = self.state.config.merge.mode;
9412 let style = self.state.config.merge.style;
9413 let facts = if is_review_run(&self.state) {
9414 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9415 let start = review_base(
9416 &repo,
9417 &self.state.config.merge.remote,
9418 &base,
9419 &self.state.base_commit,
9420 &winner.branch,
9421 )
9422 .await;
9423 branch_facts(&repo, &start, &winner.branch).await
9424 } else {
9425 None
9426 };
9427 let leaked = if is_review_run(&self.state) {
9430 leaked_subjects(&self.state, &winner.branch).await
9431 } else {
9432 Some(Vec::new())
9433 };
9434 let Some(pr) = self
9438 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9439 .await?
9440 else {
9441 return Ok(());
9442 };
9443 let message = pr.commit_message();
9444
9445 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9446 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9447 return Ok(());
9448 }
9449 let outcome = match mode {
9450 MergeMode::None => MergeOutcome {
9451 mode,
9452 ok: true,
9453 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9454 empty: false,
9455 },
9456 MergeMode::Pr | MergeMode::Local
9457 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9458 {
9459 MergeOutcome {
9460 mode,
9461 ok: false,
9462 detail: empty_candidate_detail(&self.state, &base),
9463 empty: true,
9464 }
9465 }
9466 MergeMode::Local => {
9467 let on = git::current_branch(&repo).await?;
9468 if on.as_deref() != Some(base.as_str()) {
9469 MergeOutcome {
9470 mode,
9471 ok: false,
9472 detail: format!(
9473 "{} has {} checked out, not the base branch {base}",
9474 repo.display(),
9475 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9476 ),
9477 empty: false,
9478 }
9479 } else if !git::is_clean(&repo).await? {
9480 MergeOutcome {
9481 mode,
9482 ok: false,
9483 detail: format!("{} is dirty; refusing to merge", repo.display()),
9484 empty: false,
9485 }
9486 } else {
9487 let out = match style {
9488 MergeStyle::Merge => {
9489 git::merge_no_ff(&repo, &winner.branch, &message).await?
9490 }
9491 MergeStyle::Squash => {
9492 git::merge_squash(&repo, &winner.branch, &message).await?
9493 }
9494 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9495 };
9496 MergeOutcome {
9497 mode,
9498 ok: out.ok(),
9499 detail: if out.ok() { out.stdout } else { out.stderr },
9500 empty: false,
9501 }
9502 }
9503 }
9504 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9505 self.state.pr_publication_pending = true;
9506 self.state.save()?;
9507 let out =
9508 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9509 self.state.pr_publication_pending = out.is_err();
9510 MergeOutcome {
9511 mode,
9512 ok: out.is_ok(),
9513 detail: match out {
9514 Ok(url) => url,
9515 Err(e) => format!("{e:#}"),
9516 },
9517 empty: false,
9518 }
9519 }
9520 MergeMode::Pr => {
9521 let remote = self.state.config.merge.remote.clone();
9522 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9523 let pushed = if self.state.foreign_checkout.is_some() {
9524 let refspec = format!("{delivery_head}:refs/heads/{public}");
9525 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9526 } else {
9527 git::push(&winner.worktree, &remote, &winner.branch).await?
9528 };
9529 if !pushed.ok() {
9530 MergeOutcome {
9531 mode,
9532 ok: false,
9533 detail: pushed.stderr,
9534 empty: false,
9535 }
9536 } else {
9537 if self.state.foreign_checkout.is_some() {
9538 self.state.pr_pushed_head = Some(delivery_head.clone());
9539 }
9540 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9546 let out = match pr_merge_plan(found) {
9547 PrPlan::Create => {
9548 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9549 .await
9550 }
9551 PrPlan::Adopt { url, title } => {
9552 self.state
9553 .event("merge", format!("Pr: adopted open pull request {url}"));
9554 if self.state.github_text.as_ref().is_some_and(|g| {
9555 g.resolved
9556 && g.choice == Some(GithubTextChoice::Rewrite)
9557 && g.rewrite.is_some()
9558 }) {
9559 land::set_pr_text_verbatim(
9562 &winner.worktree,
9563 &url,
9564 &pr.title,
9565 &pr.body,
9566 )
9567 .await
9568 .map(|()| url)
9569 } else {
9570 if title != pr.title
9571 && (!is_review_run(&self.state)
9572 || leaked
9573 .as_deref()
9574 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9575 && let Err(e) =
9576 if self.state.github_text.as_ref().is_some_and(|g| {
9577 g.resolved
9578 && g.title
9579 && g.choice == Some(GithubTextChoice::Original)
9580 }) {
9581 land::set_pr_title_verbatim(
9583 &winner.worktree,
9584 &url,
9585 &pr.title,
9586 )
9587 .await
9588 } else {
9589 land::set_pr_title(
9590 &mut self.state,
9591 &winner.worktree,
9592 &url,
9593 &pr.title,
9594 )
9595 .await
9596 }
9597 {
9598 tracing::warn!("could not refresh title of {url}: {e:#}");
9599 self.state
9600 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9601 }
9602 Ok(url)
9603 }
9604 }
9605 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9606 };
9607 match out {
9608 Ok(url) => MergeOutcome {
9609 mode,
9610 ok: true,
9611 detail: url,
9612 empty: false,
9613 },
9614 Err(e) => MergeOutcome {
9615 mode,
9616 ok: false,
9617 detail: e.to_string(),
9618 empty: false,
9619 },
9620 }
9621 }
9622 }
9623 };
9624
9625 self.state.status = match (mode, outcome.ok) {
9626 (MergeMode::None, _) => RunStatus::Ready,
9627 (_, true) if outcome.empty => RunStatus::Ready,
9628 (MergeMode::Pr, true) => RunStatus::Ready,
9629 (_, true) => RunStatus::Merged,
9630 (_, false) => RunStatus::Blocked,
9631 };
9632 self.state.event(
9633 "merge",
9634 format!(
9635 "{:?}: {}",
9636 mode,
9637 outcome.detail.lines().next().unwrap_or("")
9638 ),
9639 );
9640 self.state.merge = Some(outcome);
9641 self.state.save()?;
9642
9643 if self.state.config.graph.land
9649 && mode == MergeMode::Pr
9650 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9651 {
9652 self.run_land().await?;
9653 }
9654 self.settle_questions();
9659 Ok(())
9660 }
9661
9662 async fn run_land(&mut self) -> Result<()> {
9673 crate::pr_head::ensure_owner(&self.state)?;
9674 let url = self
9675 .state
9676 .pr
9677 .as_ref()
9678 .map(|pr| pr.url.clone())
9679 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9680 .unwrap_or_default();
9681 let url = url.lines().next().unwrap_or("").trim().to_owned();
9682 if !url.starts_with("http") {
9683 return Ok(());
9684 }
9685 loop {
9688 match land::land(&mut self.state, &url).await {
9689 Ok(_)
9690 if self
9691 .state
9692 .merge_repairs
9693 .last()
9694 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9695 {
9696 if self.repair_merge_review().await? {
9697 continue;
9698 }
9699 }
9700 Ok(_) if self.state.resource_block.is_some() => {
9701 return self.stop_for_resource();
9702 }
9703 Ok(pr) if self.state.parked => {
9704 let _ = pr;
9708 }
9709 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9710 self.state.status = RunStatus::Superseded;
9711 self.state.save()?;
9712 }
9713 Ok(pr) => {
9714 self.state.status = match pr.state {
9715 land::PrLifecycle::Merged => RunStatus::Merged,
9716 _ => RunStatus::Blocked,
9717 };
9718 if bump::should_release_bump(self.state.status)
9725 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9726 {
9727 self.state
9733 .event("bump", format!("release bump skipped: {e:#}"));
9734 }
9735 if self.state.status == RunStatus::Merged {
9738 crate::followup::after_merge(&mut self.state, &pr.url).await;
9739 }
9740 self.state.save()?;
9741 }
9742 Err(e) => {
9743 if e.is::<crate::repo_lock::ResourceBlocked>() {
9744 return Err(e);
9745 }
9746 self.state.status = RunStatus::Blocked;
9747 self.state.event("land", format!("gave up: {e}"));
9748 self.state.save()?;
9749 }
9750 }
9751 break;
9752 }
9753 Ok(())
9754 }
9755
9756 async fn repair_merge_review(&mut self) -> Result<bool> {
9759 use crate::run::MergeRepairPhase as Phase;
9760 crate::pr_head::ensure_owner(&self.state)?;
9761 let i = self.state.merge_repairs.len() - 1;
9762 let request = self.state.merge_repairs[i].clone();
9763 if request.phase == Phase::Fixing {
9764 let restored = match self.state.winner().cloned() {
9765 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9766 None => Err(anyhow::anyhow!("repair has no winning branch")),
9767 };
9768 let why = match restored {
9769 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9770 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9771 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9772 };
9773 self.hold_merge_repair(i, why)?;
9774 return Ok(false);
9775 }
9776 if request.phase == Phase::Reviewing && self.state.gate_ran {
9781 let winner = self
9782 .state
9783 .winner()
9784 .context("repair has no winning branch")?;
9785 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9786 if self.state.gate_head.as_deref() != Some(head.as_str())
9787 || !git::is_clean(&winner.worktree).await?
9788 {
9789 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9790 return Ok(false);
9791 }
9792 }
9793 let failure = if i >= self.state.config.graph.land_rounds
9794 || self.state.config.graph.review_rounds == 0
9795 {
9796 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9797 } else {
9798 None
9799 };
9800 if let Some(why) = failure {
9801 self.hold_merge_repair(i, why)?;
9802 return Ok(false);
9803 }
9804 if request.phase == Phase::Requested {
9805 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9806 Ok(pr) => pr,
9807 Err(e) => {
9808 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9809 return Ok(false);
9810 }
9811 };
9812 self.state.merge_repairs[i].phase = Phase::Fixing;
9813 self.state.land_approval = None;
9814 self.state.land_armed_head = None;
9815 self.state.save()?;
9816 let evidence = serde_json::to_string_pretty(&request)?;
9819 let reason = format!(
9820 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9821 );
9822 let budget = self.state.config.graph.land_rounds;
9823 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9824 match result {
9825 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9826 Ok(land::Fixed::Deferred) => {
9827 let worktree = self
9831 .state
9832 .winner()
9833 .context("repair has no winning branch")?
9834 .worktree
9835 .clone();
9836 self.graft_restore(&worktree, &request.head).await?;
9837 self.state.merge_repairs[i].phase = Phase::Requested;
9838 self.stop_for_resource()?;
9839 return Ok(false);
9840 }
9841 Ok(land::Fixed::Failed(why)) => {
9842 self.hold_merge_repair(
9843 i,
9844 format!(
9845 "repair failed: {why}; inspect the branch and run an explicit review"
9846 ),
9847 )?;
9848 return Ok(false);
9849 }
9850 Err(e) => {
9851 self.hold_merge_repair(
9852 i,
9853 format!(
9854 "repair failed: {e:#}; inspect the branch and run an explicit review"
9855 ),
9856 )?;
9857 return Ok(false);
9858 }
9859 }
9860 self.begin_merge_repair_review(i)?;
9864 }
9865 self.review_loop().await?;
9866 self.gate().await?;
9867 if self.resource_halt()? || self.state.parked {
9868 return Ok(false);
9869 }
9870 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9871 != Some(RunStatus::Gating)
9872 || !self.state.gate_allows_progress()
9873 {
9874 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9875 return Ok(false);
9876 }
9877 let winner = self
9880 .state
9881 .winner()
9882 .cloned()
9883 .context("repair has no winning branch")?;
9884 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9885 if self.state.gate_head.as_deref() != Some(after.as_str())
9886 || !git::is_clean(&winner.worktree).await?
9887 {
9888 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9889 return Ok(false);
9890 }
9891 let expected = self
9892 .state
9893 .pr_pushed_head
9894 .as_deref()
9895 .unwrap_or(&request.head)
9896 .to_owned();
9897 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9898 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9901 return Ok(false);
9902 }
9903 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9904 let refspec = format!("{after}:refs/heads/{public}");
9905 let push = git::git_raw(
9906 &winner.worktree,
9907 &["push", &lease, &self.state.config.merge.remote, &refspec],
9908 )
9909 .await;
9910 match push {
9911 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9912 Ok(out) => {
9913 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9914 return Ok(false);
9915 }
9916 Err(e) => {
9917 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9918 return Ok(false);
9919 }
9920 }
9921 self.state.merge_repairs[i].phase = Phase::Complete;
9922 self.state.status = RunStatus::Landing;
9923 self.state.event(
9924 "repair",
9925 "new review and gate completed; fresh owner merge decision required",
9926 );
9927 self.state.save()?;
9928 Ok(true)
9929 }
9930
9931 fn merge_repair_artifacts(&self) -> PathBuf {
9932 let artifacts = agent::artifacts_dir(&self.state.dir());
9933 match self
9934 .state
9935 .merge_repairs
9936 .last()
9937 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9938 {
9939 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9940 None => artifacts,
9941 }
9942 }
9943
9944 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9945 if let Some(e2e) = self.state.head_e2e.take() {
9946 self.state
9947 .verification_history
9948 .push(crate::run::VerificationEvidence {
9949 superseded_by: None,
9950 archived_at: Some(Timestamp::now()),
9951 fixes_spent: Some(self.state.gate_fixes.len()),
9952 gate_head: None,
9953 gate_ran: false,
9954 gate: Vec::new(),
9955 e2e: Some(e2e),
9956 });
9957 }
9958 self.state.reviews.clear();
9959 self.state.gate.clear();
9960 self.state.gate_ran = false;
9961 self.state.gate_head = None;
9962 self.state.gate_fixes.clear();
9963 self.state.gate_repair_e2e = None;
9964 self.state.pre_gate.clear();
9965 self.state.pre_gate_commit = None;
9966 self.state.contested_handoff = None;
9967 self.state.land_reruns = None;
9968 self.state.land_approval = None;
9969 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9970 self.state.status = RunStatus::Reviewing;
9971 self.state.save()
9972 }
9973
9974 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9975 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9976 self.state.merge_repairs[index].failure = Some(why.clone());
9977 self.state.status = RunStatus::Blocked;
9978 self.state.event("repair", why);
9979 self.state.save()
9980 }
9981
9982 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9986 if let Some(existing) = self.state.seats.get(key)
9987 && existing.agent == agent
9988 {
9989 return existing.clone();
9990 }
9991 let fresh = if self.state.seats.contains_key(key) {
9994 handover_seat(key, agent, self.state.next_seat_seed())
9995 } else {
9996 SeatState::new(key, agent, self.state.seed)
9997 };
9998 self.state.seats.insert(key.to_owned(), fresh.clone());
9999 fresh
10000 }
10001
10002 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10008 match self.state.seats.get(key) {
10009 Some(s) if s.agent != spec.id => {
10010 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10011 }
10012 _ => spec,
10013 }
10014 }
10015
10016 fn view(&self, c: &Candidate) -> CandidateView {
10018 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10019 .unwrap_or_default();
10020 let (patch, _) = blind::sanitize_patch(
10021 &format!("candidate {} patch", c.label),
10022 &raw,
10023 &self.state.config.blind,
10024 );
10025 CandidateView {
10026 label: c.label,
10027 branch: c.branch.clone(),
10028 summary: c.summary.clone(),
10029 stat: c.stat.clone(),
10030 patch,
10031 }
10032 }
10033
10034 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10036 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10037 prompt::judge(
10038 "(see above)",
10039 &views,
10040 self.roles.judges.len(),
10041 base_short,
10042 "en",
10043 )
10044 }
10045
10046 fn vote_prompt_full(
10052 &self,
10053 j: usize,
10054 labels: &[char],
10055 language: &str,
10056 candidates: &[Candidate],
10057 base_short: &str,
10058 ) -> String {
10059 let mut text = format!(
10060 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10061 prompt::final_vote(labels, language),
10062 self.state.instruction,
10063 self.candidate_block(candidates, base_short)
10064 );
10065 if let Some(own) = self
10066 .state
10067 .judgements
10068 .get(j)
10069 .filter(|r| !r.ranking.is_empty())
10070 {
10071 let reasons = own
10072 .reasons
10073 .iter()
10074 .map(|(k, v)| format!("- {k}: {v}"))
10075 .collect::<Vec<_>>()
10076 .join("\n");
10077 text.push_str(&format!(
10078 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10079 own.ranking.iter().collect::<String>(),
10080 if reasons.is_empty() {
10081 ""
10082 } else {
10083 ", because:\n"
10084 }
10085 ));
10086 }
10087 if !self.state.deliberation.is_empty() {
10088 text.push_str("\n# What was argued before this vote\n");
10089 for t in self.transcript(&[], j) {
10090 text.push_str(&format!(
10091 "\n## {}{}\n\n{}\n",
10092 t.who,
10093 if t.is_self { " (you)" } else { "" },
10094 t.body.trim()
10095 ));
10096 }
10097 }
10098 text
10099 }
10100
10101 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10108 let mut turns = Vec::new();
10109 for j in &self.state.judgements {
10110 if j.ranking.is_empty() {
10111 continue;
10112 }
10113 let reasons = j
10114 .reasons
10115 .iter()
10116 .map(|(k, v)| format!("- {k}: {v}"))
10117 .collect::<Vec<_>>()
10118 .join("\n");
10119 turns.push(Turn {
10120 who: format!("Judge {} (opening ranking)", j.judge),
10121 is_self: j.judge == self_idx + 1,
10122 body: format!(
10123 "Ranked {}{}{reasons}",
10124 j.ranking.iter().collect::<String>(),
10125 if reasons.is_empty() {
10126 ""
10127 } else {
10128 ", because:\n"
10129 }
10130 ),
10131 });
10132 }
10133 for t in self
10134 .state
10135 .deliberation
10136 .iter()
10137 .flat_map(|r| r.turns.iter())
10138 .chain(current)
10139 {
10140 turns.push(Turn {
10141 who: format!("Judge {}", t.judge),
10142 is_self: t.judge == self_idx + 1,
10143 body: t.body.clone(),
10144 });
10145 }
10146 turns
10147 }
10148}
10149
10150fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10152 agent::has_session(spec.kind, seat, sessions)
10153}
10154
10155fn next_untried_in_roster<'a>(
10176 roster: &'a [AgentSpec],
10177 start: usize,
10178 tried: &BTreeSet<String>,
10179) -> Option<&'a AgentSpec> {
10180 roster
10181 .get(start + 1..)?
10182 .iter()
10183 .find(|s| !tried.contains(&s.id))
10184}
10185
10186fn pick_successor<'a>(
10207 roster: &'a [AgentSpec],
10208 start: usize,
10209 tried: &BTreeSet<String>,
10210 carried: Option<&BTreeSet<String>>,
10211 others: &BTreeSet<String>,
10212) -> Option<&'a AgentSpec> {
10213 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10214 roster
10215 .get(start + 1..)
10216 .and_then(|tail| {
10217 tail.iter()
10218 .filter(free)
10219 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10220 })
10221 .or_else(|| {
10222 carried
10223 .is_some()
10224 .then(|| roster.iter().find(free))
10225 .flatten()
10226 })
10227 .or_else(|| match carried {
10228 Some(c) => next_for_seat(roster, start, tried, c),
10229 None => next_untried_in_roster(roster, start, tried),
10230 })
10231}
10232
10233fn next_for_seat<'a>(
10243 roster: &'a [AgentSpec],
10244 start: usize,
10245 round_tried: &BTreeSet<String>,
10246 carried_failed: &BTreeSet<String>,
10247) -> Option<&'a AgentSpec> {
10248 roster
10249 .get(start + 1..)?
10250 .iter()
10251 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10252 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10253}
10254
10255fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10261 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10262 return spec;
10263 };
10264 let ok = |id: &str| !h.failed.contains(id);
10265 if let Some(last) = h.last_ok.as_deref()
10266 && ok(last)
10267 && let Some(s) = roster.iter().find(|s| s.id == last)
10268 {
10269 return s.clone();
10270 }
10271 if ok(&spec.id) {
10272 return spec;
10273 }
10274 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10275 roster
10276 .get(from + 1..)
10277 .into_iter()
10278 .flatten()
10279 .chain(roster.iter())
10280 .find(|s| ok(&s.id))
10281 .cloned()
10282 .unwrap_or(spec)
10283}
10284
10285pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10289 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10290}
10291
10292const TIMED_OUT: &str = "timed out";
10295
10296impl FailClass {
10297 fn of(out: &AgentOutcome) -> Option<Self> {
10299 match out {
10300 AgentOutcome::Ok(_) => None,
10301 AgentOutcome::Quota(_) => Some(Self::Quota),
10302 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10303 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10304 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10305 AgentOutcome::ResourceBlocked(_) => None,
10308 }
10309 }
10310
10311 fn stem_word(&self) -> &'static str {
10313 match self {
10314 Self::Quota => "quota",
10315 _ => "handover",
10316 }
10317 }
10318}
10319
10320fn failure_signature(msg: &str) -> String {
10324 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10325 if let Some(code) = line
10328 .strip_prefix("process exited with no usable output (exit code ")
10329 .and_then(|s| s.strip_suffix(')'))
10330 .and_then(|s| s.parse::<i32>().ok())
10331 {
10332 line = format!("exited with some({code}) and no usable output");
10333 } else if matches!(
10334 line.as_str(),
10335 "process exited with no usable output (no exit code reported)"
10336 | "process exited with no usable output (exit code unavailable)"
10337 ) {
10338 line = "exited with none and no usable output".to_owned();
10339 }
10340 let mut out = Vec::new();
10341 for word in line.split_whitespace() {
10342 if word.contains('/') || word.contains('\\') {
10343 out.push("<path>".to_owned());
10344 continue;
10345 }
10346 let mut w = String::new();
10347 let mut in_digits = false;
10348 for c in word.chars() {
10349 if c.is_ascii_digit() {
10350 if !in_digits {
10351 w.push('#');
10352 }
10353 in_digits = true;
10354 } else {
10355 in_digits = false;
10356 w.push(c);
10357 }
10358 }
10359 out.push(w);
10360 }
10361 out.join(" ").chars().take(120).collect()
10362}
10363
10364fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10371 match cur {
10372 FailClass::Quota | FailClass::Timeout => true,
10373 FailClass::Other(_) => prev != Some(cur),
10374 }
10375}
10376
10377fn fail_reason(out: &AgentOutcome) -> String {
10379 match out {
10380 AgentOutcome::Ok(_) => String::new(),
10381 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10382 AgentOutcome::Dropped(o) => format!(
10383 "the CLI dropped the stream ({})",
10384 o.dropped
10385 .as_ref()
10386 .map(|d| d.why.as_str())
10387 .unwrap_or("it ended without delivering its answer")
10388 ),
10389 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10390 e.lines().next().unwrap_or("").chars().take(160).collect()
10391 }
10392 }
10393}
10394
10395pub(crate) fn record_handover(
10398 state: &mut RunState,
10399 node: &str,
10400 seat: &str,
10401 from: &str,
10402 to: &str,
10403 class: &FailClass,
10404 reason: &str,
10405) {
10406 let message = if *class == FailClass::Quota {
10407 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10408 } else {
10409 format!("{seat}: handed over {from} -> {to} ({reason})")
10410 };
10411 state.event(node, message);
10412 state.handovers.push(Handover {
10413 at: Timestamp::now(),
10414 node: node.to_owned(),
10415 seat: seat.to_owned(),
10416 from: from.to_owned(),
10417 to: to.to_owned(),
10418 reason: reason.to_owned(),
10419 });
10420}
10421
10422fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10436 commands.iter().any(|c| c.exit_code.is_none())
10437}
10438
10439fn verified_noop_claim(
10452 usable: bool,
10453 commands: &[agent::CommandEvidence],
10454 text: &str,
10455) -> Option<String> {
10456 (usable && !has_unconfirmed_command(commands))
10457 .then(|| verdict::verified_noop(text))
10458 .flatten()
10459}
10460
10461fn short(commit: &str) -> String {
10462 commit.chars().take(7).collect()
10463}
10464
10465fn make_executable(path: &Path) -> Result<()> {
10466 #[cfg(unix)]
10467 {
10468 use std::os::unix::fs::PermissionsExt as _;
10469 let mut perms = std::fs::metadata(path)?.permissions();
10470 perms.set_mode(0o755);
10471 std::fs::set_permissions(path, perms)?;
10472 }
10473 #[cfg(not(unix))]
10474 {
10475 let _ = path;
10476 }
10477 Ok(())
10478}
10479
10480struct WaveCtx<'a> {
10487 run: &'a str,
10490 node: &'a str,
10492 prompts: &'a Prompts,
10493 cache: Option<&'a Path>,
10495 round: Option<usize>,
10498 carry_seats: bool,
10502 pauses: Option<(&'a Pause, &'a Pause)>,
10505}
10506
10507fn repository_wait(
10509 state: &RunState,
10510 pauses: Option<(&Pause, &Pause)>,
10511) -> crate::repo_lock::WaitPolicy {
10512 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10513 let inherited = crate::repo_lock::WaitPolicy::current();
10514 crate::repo_lock::WaitPolicy::new(
10515 Duration::from_secs(state.config.graph.timeout_resource_wait),
10516 move || {
10517 inherited.cancelled()
10518 || pauses
10519 .as_ref()
10520 .is_some_and(|(a, b)| a.parked() || b.parked())
10521 },
10522 )
10523}
10524
10525fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10526 if error.is::<crate::repo_lock::ResourceBlocked>() {
10527 state.resource_block = Some(crate::run::ResourceBlock {
10528 cause: crate::run::ResourceBlockCause::Other,
10529 node: "repository".into(),
10530 kind: if error.to_string().contains("disk pressure") {
10531 "disk pressure"
10532 } else {
10533 "repository coordination"
10534 }
10535 .into(),
10536 detail: error.to_string(),
10537 at: Timestamp::now(),
10538 });
10539 state.event("resource", error.to_string());
10540 }
10541}
10542
10543async fn coordinate_operation(
10544 state: &mut RunState,
10545 cwd: &Path,
10546 wait: &crate::repo_lock::WaitPolicy,
10547) -> Result<Option<crate::repo_lock::SeatLease>> {
10548 let result =
10549 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10550 .await;
10551 if let Err(e) = &result {
10552 record_repository_block(state, e);
10553 }
10554 result
10555}
10556
10557async fn run_one(
10559 job: SeatJob,
10560 sem: Arc<Semaphore>,
10561 ctx: &WaveCtx<'_>,
10562 state: &mut RunState,
10563 attempt: usize,
10564) -> (SeatState, AgentOutcome) {
10565 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10566 .await
10567 .pop()
10568 .expect("one job in, one result out");
10569 (seat, out)
10570}
10571
10572async fn wave(
10578 jobs: Vec<SeatJob>,
10579 sem: Arc<Semaphore>,
10580 ctx: &WaveCtx<'_>,
10581 state: &mut RunState,
10582 attempt: usize,
10583) -> Vec<(usize, SeatState, AgentOutcome)> {
10584 let WaveCtx {
10585 run,
10586 node,
10587 prompts,
10588 cache,
10589 round,
10590 carry_seats: _,
10591 pauses,
10592 } = *ctx;
10593 for job in &jobs {
10594 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10595 }
10596 if let Err(e) = state.save() {
10597 tracing::warn!("could not persist in-progress seats: {e:#}");
10602 }
10603 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10619 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10623 let cache_guard = if let Some(cache_dir) = cache {
10624 if jobs_had_a_writer {
10625 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10626 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10627 match acquire_cache_lease(
10628 state,
10629 cache_dir,
10630 &owner,
10631 resource_limit,
10632 node,
10633 node,
10634 &cancel,
10635 )
10636 .await
10637 {
10638 Ok(Some(g)) => Some(g),
10639 Ok(None) => {
10642 if let Some(block) = &mut state.resource_block {
10643 block.cause = crate::run::ResourceBlockCause::Cancelled {
10644 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10645 };
10646 }
10647 return block_wave(
10648 jobs,
10649 state,
10650 node,
10651 attempt,
10652 "parked while waiting for the shared build cache",
10653 );
10654 }
10655 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10658 Err(e) => {
10659 let why = format!("{e:#}");
10663 state.event(
10664 "resource",
10665 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10666 );
10667 return block_wave(jobs, state, node, attempt, &why);
10668 }
10669 }
10670 } else {
10671 None
10672 }
10673 } else {
10674 None
10675 };
10676 state.seats_spawning(node, attempt);
10679 if let Err(e) = state.save() {
10680 tracing::warn!("could not persist the start of a wave: {e:#}");
10681 }
10682 let slot_limit = resource_limit
10686 + jobs
10687 .iter()
10688 .map(|j| j.timeout)
10689 .max()
10690 .unwrap_or_default()
10691 .saturating_mul(jobs.len() as u32);
10692 let mut set = tokio::task::JoinSet::new();
10693 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10696 let overlay = prompts.overlay(node);
10697 let repository_wait = repository_wait(state, pauses);
10698 for (i, mut job) in jobs.into_iter().enumerate() {
10699 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10700 if cache.is_some() {
10701 job.prompt.push('\n');
10702 job.prompt
10703 .push_str(&prompt::build_cache_note(node, job.allow_write));
10704 }
10705 let sem = Arc::clone(&sem);
10706 let run = run.to_owned();
10707 let node = node.to_owned();
10708 let attachments = if node == "implement" {
10711 state.attachments.clone()
10712 } else {
10713 Vec::new()
10714 };
10715 let cache = cache
10726 .filter(|_| job.allow_write && cache_guard.is_some())
10727 .map(Path::to_path_buf);
10728 let spawned_tx = spawned_tx.clone();
10729 let repository_wait = repository_wait.clone();
10730 let coordination = state.config.coordination.clone();
10731 let fetched_base = state.fetched_base.clone();
10732 set.spawn(async move {
10733 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10737 Ok(p) => p,
10738 Err(_) => {
10739 return (
10740 i,
10741 job.seat,
10742 AgentOutcome::ResourceBlocked(format!(
10743 "no parallel slot became free within {}s",
10744 slot_limit.as_secs()
10745 )),
10746 false,
10747 );
10748 }
10749 };
10750 spawned_tx.send(job.seat.key.clone()).ok();
10751 let mut seat = job.seat;
10752 let out = repository_wait
10753 .scope(agent::invoke(
10754 &job.spec,
10755 &mut seat,
10756 &Invocation {
10757 coordination: coordination.as_ref(),
10758 fetched_base: fetched_base.as_ref(),
10759 cwd: &job.cwd,
10760 prompt: &job.prompt,
10761 timeout: job.timeout,
10762 allow_write: job.allow_write,
10763 sessions: job.sessions,
10764 artifacts: &job.artifacts,
10765 stem: &job.stem,
10766 run: &run,
10767 node: &node,
10768 cache_dir: cache.as_deref(),
10769 attachments: &attachments,
10770 writable: &[],
10771 },
10772 ))
10773 .await;
10774 let interrupted = out
10775 .as_ref()
10776 .err()
10777 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10778 let out = match out {
10779 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10780 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10781 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10789 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10790 Ok(o) => AgentOutcome::Failed(format!(
10791 "process exited with no usable output ({})",
10792 crate::notices::exit_status(o.exit_code)
10793 )),
10794 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10795 AgentOutcome::ResourceBlocked(e.to_string())
10796 }
10797 Err(e) => AgentOutcome::Failed(e.to_string()),
10798 };
10799 (i, seat, out, interrupted)
10800 });
10801 }
10802 drop(spawned_tx);
10803 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10804 loop {
10805 let joined = tokio::select! {
10806 Some(key) = spawned_rx.recv() => {
10807 state.seat_spawned(&key);
10809 if let Err(e) = state.save() {
10810 tracing::warn!("could not persist a seat's start: {e:#}");
10811 }
10812 continue;
10813 }
10814 joined = set.join_next() => joined,
10815 };
10816 let Some(joined) = joined else {
10817 break;
10818 };
10819 let (i, seat, out, interrupted) = match joined {
10820 Ok(v) => v,
10821 Err(e) => {
10825 tracing::error!("agent task panicked: {e}");
10826 continue;
10827 }
10828 };
10829 state.seat_finished(&seat.key);
10830 if interrupted
10831 && node == "implement"
10832 && let Some(label) = state
10833 .candidates
10834 .iter()
10835 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10836 .map(|candidate| candidate.label)
10837 {
10838 state.candidate_preservation.insert(
10839 label,
10840 "interrupted implementation awaiting persistence".into(),
10841 );
10842 state.interrupted_implementations.insert(label);
10843 }
10844 if let AgentOutcome::ResourceBlocked(why) = &out
10845 && state.resource_block.is_none()
10846 {
10847 state.resource_block = Some(crate::run::ResourceBlock {
10848 cause: crate::run::ResourceBlockCause::Other,
10849 node: node.to_owned(),
10850 kind: if why.contains("disk pressure") {
10851 "disk pressure"
10852 } else if why.contains("repository coordination") {
10853 "repository coordination"
10854 } else {
10855 "parallel slot"
10856 }
10857 .to_owned(),
10858 detail: why.clone(),
10859 at: Timestamp::now(),
10860 });
10861 let activity = if interrupted {
10862 "agent execution interrupted"
10863 } else {
10864 "no agent was started"
10865 };
10866 state.event("resource", format!("{node}: {activity} - {why}"));
10867 }
10868 record_jobs(state, node, round, &seat.key, &out);
10869 if let Err(e) = state.save() {
10870 tracing::warn!("could not persist a seat's completion: {e:#}");
10871 }
10872 if collected.len() <= i {
10873 collected.resize_with(i + 1, || None);
10874 }
10875 collected[i] = Some((i, seat, out));
10876 }
10877 if state
10883 .active
10884 .values()
10885 .any(|a| a.node == node && a.attempt == attempt)
10886 {
10887 state
10888 .active
10889 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10890 if let Err(e) = state.save() {
10891 tracing::warn!("could not persist the end of a wave: {e:#}");
10892 }
10893 }
10894 if let Some(cache_dir) = cache
10901 && jobs_had_a_writer
10902 {
10903 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10904 }
10905 if let Some(guard) = cache_guard {
10906 guard.release();
10907 }
10908 collected.into_iter().flatten().collect()
10909}
10910
10911fn record_jobs(
10922 state: &mut RunState,
10923 node: &str,
10924 round: Option<usize>,
10925 seat: &str,
10926 out: &AgentOutcome,
10927) {
10928 let commands: &[agent::CommandEvidence] = match out {
10929 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10930 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10931 };
10932 let checked_at = Timestamp::now();
10933 for c in commands {
10934 state.jobs.push(JobRecord {
10935 node: node.to_owned(),
10936 round,
10937 seat: seat.to_owned(),
10938 id: c.id.clone(),
10939 description: c.description.clone(),
10940 checked_at,
10941 status: match c.exit_code {
10942 Some(0) => JobStatus::Completed,
10943 Some(_) => JobStatus::Failed,
10944 None => JobStatus::Unknown,
10945 },
10946 exit_code: c.exit_code,
10947 result_summary: c.result_summary.clone(),
10948 source: c.source.clone(),
10949 });
10950 }
10951}
10952
10953fn round_is_clean(
10974 blocking: usize,
10975 e2e_ok: bool,
10976 answered: usize,
10977 expected: usize,
10978 quota_missing: usize,
10979 policy: IncompleteReviewPolicy,
10980) -> bool {
10981 if blocking != 0 || !e2e_ok {
10982 return false;
10983 }
10984 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10985 return true;
10986 }
10987 answered > 0 && expected - answered <= quota_missing
10988}
10989
10990pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11014 if max_rounds == 0
11015 || reviews.last().is_some_and(|r| {
11016 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11017 })
11018 {
11019 return Some(RunStatus::Gating);
11020 }
11021 let last = reviews.last()?;
11022 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11023 if reviews.len() < max_rounds && !stagnant {
11024 return None;
11025 }
11026 if last.incomplete() && last.blocking == 0 {
11027 return Some(RunStatus::Blocked);
11028 }
11029 if last.e2e_status() == E2eStatus::ResourceBlocked
11030 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11031 {
11032 return None;
11033 }
11034 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11035 RunStatus::Gating
11036 } else {
11037 RunStatus::Blocked
11038 })
11039}
11040
11041pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11056 if nudged {
11057 (full / 4).max(Duration::from_secs(120)).min(full)
11058 } else {
11059 full
11060 }
11061}
11062
11063#[allow(clippy::too_many_arguments)]
11092async fn ask_json_wave<T>(
11093 jobs: Vec<SeatJob>,
11094 sem: Arc<Semaphore>,
11095 retries: usize,
11096 roster: &[AgentSpec],
11097 ctx: &WaveCtx<'_>,
11098 losses: &mut Vec<QuotaLoss>,
11099 state: &mut RunState,
11100 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11101) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11102where
11103 T: serde::de::DeserializeOwned + Send + 'static,
11104{
11105 ask_wave_with(
11106 jobs,
11107 sem,
11108 retries,
11109 roster,
11110 ctx,
11111 losses,
11112 state,
11113 &|text: &str| {
11114 let v = verdict::extract_json::<T>(text)?;
11115 validate(&v)?;
11116 Ok(v)
11117 },
11118 )
11119 .await
11120}
11121
11122#[allow(clippy::too_many_arguments)]
11131async fn ask_wave_with<T>(
11132 jobs: Vec<SeatJob>,
11133 sem: Arc<Semaphore>,
11134 retries: usize,
11135 roster: &[AgentSpec],
11136 ctx: &WaveCtx<'_>,
11137 losses: &mut Vec<QuotaLoss>,
11138 state: &mut RunState,
11139 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11140) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11141where
11142 T: Send + 'static,
11143{
11144 let n = jobs.len();
11145 let originals: Vec<SeatJob> = jobs;
11146 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11147 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11148 let mut nudges: Vec<usize> = vec![0; n];
11155 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11159 let mut tried: Vec<BTreeSet<String>> = specs
11160 .iter()
11161 .map(|s| BTreeSet::from([s.id.clone()]))
11162 .collect();
11163 let starts: Vec<usize> = specs
11164 .iter()
11165 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11166 .collect();
11167 let carry = ctx.carry_seats && !roster.is_empty();
11168 let carried: Vec<BTreeSet<String>> = originals
11171 .iter()
11172 .map(|j| {
11173 state
11174 .seat_history
11175 .get(&j.seat.key)
11176 .filter(|_| carry)
11177 .map(|h| h.failed.clone())
11178 .unwrap_or_default()
11179 })
11180 .collect();
11181 let mut prev: Vec<Option<FailClass>> = originals
11182 .iter()
11183 .map(|j| {
11184 state
11185 .seat_history
11186 .get(&j.seat.key)
11187 .filter(|_| carry)
11188 .and_then(|h| h.last_fail.clone())
11189 })
11190 .collect();
11191 let next_agent =
11192 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11193 let others: BTreeSet<String> = specs
11194 .iter()
11195 .enumerate()
11196 .filter(|(j, _)| *j != i)
11197 .map(|(_, s)| s.id.clone())
11198 .collect();
11199 pick_successor(
11200 roster,
11201 starts[i],
11202 tried,
11203 carry.then(|| &carried[i]),
11204 &others,
11205 )
11206 .cloned()
11207 };
11208 let mut fresh: Vec<Option<String>> = vec![None; n];
11209 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11210 let mut pending: Vec<usize> = (0..n).collect();
11211
11212 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11217 for round in 0..max_rounds {
11218 if pending.is_empty() {
11219 break;
11220 }
11221 let mut batch = Vec::with_capacity(pending.len());
11222 let mut renudged: Vec<&str> = Vec::new();
11223 for &i in &pending {
11224 let src = &originals[i];
11225 let full: &str = match &src.handover {
11230 Some(h) if specs[i].id != src.spec.id => h,
11231 _ => &src.prompt,
11232 };
11233 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11236 (
11237 full.to_owned(),
11238 src.timeout,
11239 format!("{}-{word}-{}", src.stem, specs[i].id),
11240 )
11241 } else if nudges[i] == 0 {
11242 (full.to_owned(), src.timeout, src.stem.clone())
11243 } else {
11244 renudged.push(src.seat.key.as_str());
11245 let why = done[i]
11246 .as_ref()
11247 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11248 .unwrap_or_else(|| "no parsable answer".to_owned());
11249 let nudge = prompt::nudge(&why);
11250 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11251 let prompt = if nudged {
11252 nudge
11253 } else {
11254 format!("{full}\n\n---\n\n{nudge}")
11255 };
11256 (
11257 prompt,
11258 retry_budget(src.timeout, nudged),
11259 format!("{}-retry{}", src.stem, nudges[i]),
11260 )
11261 };
11262 batch.push(SeatJob {
11263 spec: specs[i].clone(),
11264 seat: seats[i].clone(),
11265 cwd: src.cwd.clone(),
11266 prompt,
11267 timeout,
11268 allow_write: src.allow_write,
11269 sessions: src.sessions,
11270 artifacts: src.artifacts.clone(),
11271 stem,
11272 handover: None,
11273 });
11274 }
11275
11276 if !renudged.is_empty() {
11277 state.event(
11278 ctx.node,
11279 format!("retry {round}: re-asking {}", renudged.join(", ")),
11280 );
11281 }
11282 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11283 let mut still = Vec::new();
11284 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11285 seats[i] = seat;
11286 if let AgentOutcome::ResourceBlocked(why) = &out {
11289 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11290 continue;
11291 }
11292 let class = FailClass::of(&out);
11293 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11296 && nudges[i] < retries
11297 && !roster.is_empty();
11298 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11299 let next = should_hand_over(prev[i].as_ref(), &cur)
11300 .then(|| next_agent(i, &tried[i], &specs))
11301 .flatten();
11302 if carry {
11303 let h = state
11304 .seat_history
11305 .entry(originals[i].seat.key.clone())
11306 .or_default();
11307 h.failed.insert(specs[i].id.clone());
11308 h.last_fail = Some(cur.clone());
11309 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11310 h.last_ok = None;
11311 }
11312 if let Err(e) = state.save() {
11315 tracing::warn!("could not persist a seat's failure history: {e:#}");
11316 }
11317 }
11318 if let Some(next) = next {
11319 record_handover(
11320 state,
11321 ctx.node,
11322 &originals[i].seat.key,
11323 &specs[i].id,
11324 &next.id,
11325 &cur,
11326 &fail_reason(&out),
11327 );
11328 tried[i].insert(next.id.clone());
11329 prev[i] = Some(cur.clone());
11330 seats[i] =
11331 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11332 specs[i] = next;
11333 fresh[i] = Some(cur.stem_word().to_owned());
11334 nudges[i] = 0;
11335 done[i] = Some(Err(anyhow::anyhow!(
11336 "handed over after: {}",
11337 fail_reason(&out)
11338 )));
11339 still.push(i);
11340 continue;
11341 }
11342 }
11343 if carry
11344 && !nudge_first
11345 && let Some(cur) = class.clone()
11346 {
11347 let h = state
11349 .seat_history
11350 .entry(originals[i].seat.key.clone())
11351 .or_default();
11352 h.failed.insert(specs[i].id.clone());
11353 h.last_fail = Some(cur);
11354 }
11355 let parsed = match out {
11356 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11357 AgentOutcome::Quota(o) => {
11358 last_quota[i] = o.quota.clone();
11359 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11360 }
11361 AgentOutcome::Dropped(o) => {
11365 let why = o
11366 .dropped
11367 .as_ref()
11368 .map(|d| d.why.as_str())
11369 .unwrap_or("the CLI ended the stream without delivering its answer");
11370 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11371 }
11372 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11373 Err(anyhow::anyhow!(e))
11374 }
11375 };
11376 let quota = class == Some(FailClass::Quota);
11377 let failed = parsed.is_err();
11378 if !failed && matches!(ctx.node, "judge" | "vote") {
11379 state
11382 .quota
11383 .retain(|loss| loss.seat != originals[i].seat.key);
11384 }
11385 done[i] = Some(parsed);
11386 if carry && !failed {
11387 let h = state
11388 .seat_history
11389 .entry(originals[i].seat.key.clone())
11390 .or_default();
11391 h.failed.remove(&specs[i].id);
11392 h.last_ok = Some(specs[i].id.clone());
11393 h.last_fail = None;
11394 }
11395 let agent_failure = class.is_some()
11403 && !nudge_first
11404 && !roster.is_empty()
11405 && next_agent(i, &tried[i], &specs).is_some();
11406 if failed && !quota && !agent_failure && nudges[i] < retries {
11407 nudges[i] += 1;
11408 still.push(i);
11409 }
11410 }
11411 pending = still;
11412 }
11413
11414 for (i, q) in last_quota.into_iter().enumerate() {
11417 if let Some(quota) = q {
11418 losses.push(QuotaLoss::from_report(
11419 originals[i].seat.key.clone(),
11420 ctx.node,
11421 Some("a),
11422 ));
11423 }
11424 }
11425
11426 if state.status != RunStatus::Stalled
11429 && ctx.node != "deliberate"
11430 && n > 0
11431 && losses.len() == n
11432 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11433 {
11434 block_machine_outage(state, ctx.node, losses);
11435 state.quota.append(losses);
11436 }
11437
11438 seats
11439 .into_iter()
11440 .zip(done)
11441 .zip(nudges)
11442 .map(|((seat, res), attempts)| {
11443 (
11444 seat,
11445 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11446 attempts,
11447 )
11448 })
11449 .collect()
11450}
11451
11452pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11455 let mut auth: Vec<_> = losses
11456 .iter()
11457 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11458 .map(|loss| loss.agent.clone())
11459 .collect();
11460 auth.sort();
11461 auth.dedup();
11462 let (kind, detail) = if auth.is_empty() {
11463 (
11464 "agent usage limit",
11465 "All seats reached their agent usage limits; resume after usage is available"
11466 .to_owned(),
11467 )
11468 } else {
11469 (
11470 "agent authentication",
11471 format!(
11472 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11473 auth.join(", ")
11474 ),
11475 )
11476 };
11477 state.resource_block = Some(crate::run::ResourceBlock {
11478 node: node.to_owned(),
11479 kind: kind.to_owned(),
11480 cause: crate::run::ResourceBlockCause::Other,
11481 detail: detail.clone(),
11482 at: Timestamp::now(),
11483 });
11484 state.event(node, detail);
11485}
11486
11487fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11489 results
11490 .iter()
11491 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11492}
11493
11494fn block_wave(
11498 jobs: Vec<SeatJob>,
11499 state: &mut RunState,
11500 node: &str,
11501 attempt: usize,
11502 why: &str,
11503) -> Vec<(usize, SeatState, AgentOutcome)> {
11504 state.resource_wait = None;
11505 state
11506 .active
11507 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11508 if let Err(e) = state.save() {
11509 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11510 }
11511 jobs.into_iter()
11512 .enumerate()
11513 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11514 .collect()
11515}
11516
11517async fn acquire_cache_lease(
11530 state: &mut RunState,
11531 cache_dir: &Path,
11532 owner: &crate::cache::Owner,
11533 budget: Duration,
11534 node: &str,
11535 context: &str,
11536 cancel: &(dyn Fn() -> bool + Sync),
11537) -> Result<Option<crate::cache::Guard>> {
11538 let home = crate::run::home();
11539 let started = Instant::now();
11540 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11541 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11542 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11543 Err(e) => {
11544 state.event(
11545 "verify",
11546 format!("{context}: could not check the shared build cache: {e:#}"),
11547 );
11548 if let Err(e2) = state.save() {
11549 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11550 }
11551 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11552 }
11553 };
11554 state.event(
11555 "verify",
11556 format!(
11557 "{context}: waiting for the shared build cache at {} ({})",
11558 cache_dir.display(),
11559 busy.describe()
11560 ),
11561 );
11562 state.resource_wait = Some(crate::run::ResourceWait {
11565 node: context.to_owned(),
11566 kind: "shared build cache".to_owned(),
11567 holder: busy.describe(),
11568 since: Timestamp::now(),
11569 limit_secs: budget.as_secs(),
11570 });
11571 if let Err(e) = state.save() {
11572 tracing::warn!("could not persist a cache wait: {e:#}");
11573 }
11574 let remaining = budget.saturating_sub(started.elapsed());
11575 let waited = crate::cache::wait_for_unless(
11576 &home,
11577 cache_dir,
11578 owner,
11579 remaining,
11580 Duration::from_secs(5),
11581 cancel,
11582 )
11583 .await;
11584 state.resource_wait = None;
11585 let cause = match &waited {
11586 Ok(Some(_)) => None,
11587 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11588 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11589 Some(crate::run::ResourceBlockCause::Timeout {
11590 waited_secs: started.elapsed().as_secs(),
11591 limit_secs: budget.as_secs(),
11592 })
11593 }
11594 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11595 };
11596 if let Some(cause) = cause {
11597 let detail = match &waited {
11598 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11599 Err(error) => format!("{error:#}"),
11600 };
11601 state.resource_block = Some(crate::run::ResourceBlock {
11602 cause,
11603 node: node.to_owned(),
11604 kind: "shared build cache".into(),
11605 detail,
11606 at: Timestamp::now(),
11607 });
11608 state.event(
11609 "resource",
11610 state.resource_block.as_ref().unwrap().description(),
11611 );
11612 }
11613 if let Err(error) = state.save() {
11614 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11615 }
11616 waited
11617}
11618
11619#[derive(Debug)]
11624struct CacheCheckError(String);
11625
11626impl std::fmt::Display for CacheCheckError {
11627 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11628 f.write_str(&self.0)
11629 }
11630}
11631
11632impl std::error::Error for CacheCheckError {}
11633
11634fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11636 state
11637 .resource_block
11638 .get_or_insert_with(|| crate::run::ResourceBlock {
11639 cause: crate::run::ResourceBlockCause::Other,
11640 node: node.to_owned(),
11641 kind: "shared build cache".into(),
11642 detail: format!("{error:#}"),
11643 at: Timestamp::now(),
11644 });
11645 if let Err(error) = state.save() {
11646 tracing::warn!("could not persist cache resource block: {error:#}");
11647 }
11648}
11649
11650#[allow(clippy::too_many_arguments)]
11671async fn with_cache_lease<'s, F, Fut>(
11672 state: &'s mut RunState,
11673 cache_dir: Option<&Path>,
11674 node: &str,
11675 seat: &str,
11676 worktree: &Path,
11677 head: &str,
11678 budget: Duration,
11679 context: &str,
11680 wait: &crate::repo_lock::WaitPolicy,
11681 body: F,
11682) -> Result<(Vec<CommandOutcome>, bool)>
11683where
11684 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11685 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11686{
11687 let started = Instant::now();
11690 let home = crate::run::home();
11691 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11692 let guard = if let Some(cache_dir) = cache_dir {
11693 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11694 match acquire_cache_lease(
11695 state,
11696 cache_dir,
11697 &owner,
11698 resource_limit,
11699 node,
11700 context,
11701 &|| wait.cancelled(),
11702 )
11703 .await
11704 .and_then(|g| {
11705 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11706 }) {
11707 Ok(g) => Some(g),
11708 Err(e) => {
11709 record_cache_error(state, node, &e);
11710 return Ok((
11711 vec![CommandOutcome {
11712 command: "(waiting for the shared build cache)".into(),
11713 code: None,
11714 output_tail: e.to_string(),
11715 duration_ms: started.elapsed().as_millis() as u64,
11716 resource_blocked: true,
11717 }],
11718 false,
11719 ));
11720 }
11721 }
11722 } else {
11723 None
11724 };
11725 let _coordination = coordinate_operation(state, worktree, wait).await?;
11726 if let Some(cache_dir) = cache_dir {
11727 let identity = crate::cache::Identity::new(worktree, head);
11728 if let Err(e) = crate::cache::ensure_fresh(
11729 &home,
11730 cache_dir,
11731 &identity,
11732 &state.config.verify.cargo_manifests(),
11733 ) {
11734 record_cache_error(state, node, &e);
11735 state.event(
11736 "verify",
11737 format!(
11738 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11739 worktree.display(),
11740 short(head)
11741 ),
11742 );
11743 if let Some(guard) = guard {
11744 guard.release();
11745 }
11746 return Ok((
11747 vec![CommandOutcome {
11748 command: "(confirming the shared build cache is fresh)".into(),
11749 code: None,
11750 output_tail: e.to_string(),
11751 duration_ms: started.elapsed().as_millis() as u64,
11752 resource_blocked: true,
11753 }],
11754 false,
11755 ));
11756 }
11757 }
11758 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11759 if !timed_out_pids.is_empty() {
11760 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11761 }
11762 drop(_coordination);
11765 if let Some(guard) = guard {
11766 guard.release();
11767 }
11768 Ok((outcomes, retried))
11769}
11770
11771async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11783 wait_for_pids_with(
11784 pids,
11785 crate::proc::pid_alive,
11786 LEASE_RELEASE_POLL,
11787 LEASE_RELEASE_MAX_WAIT,
11788 )
11789 .await;
11790}
11791
11792async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11798 pids: &[u32],
11799 alive: F,
11800 poll: Duration,
11801 max_wait: Duration,
11802) {
11803 let deadline = Instant::now() + max_wait;
11804 loop {
11805 if pids.iter().all(|&pid| !alive(pid)) {
11806 return;
11807 }
11808 if Instant::now() >= deadline {
11809 return;
11810 }
11811 tokio::time::sleep(poll).await;
11812 }
11813}
11814
11815fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11822 outcomes.iter().any(|o| o.resource_blocked)
11823}
11824
11825pub(crate) enum GateFix {
11827 Retry,
11829 Stop,
11832 Defer,
11835}
11836
11837pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11845 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11846 red.peek().is_some()
11847 && red.all(|o| {
11848 !o.resource_blocked
11849 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11850 && !o.output_tail.trim().is_empty()
11851 })
11852}
11853
11854fn e2e_outcome_label(o: &CommandOutcome) -> String {
11858 if o.ok() {
11859 return "pass".to_owned();
11860 }
11861 let reason = if o.build_failed() {
11862 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11863 } else {
11864 format!("FAIL ({:?})", o.code)
11865 };
11866 format!(
11867 "{reason}\n{}",
11868 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11869 )
11870}
11871
11872async fn run_e2e_with_retry(
11880 state: &mut RunState,
11881 shell: &[String],
11882 commands: &[String],
11883 worktree: &Path,
11884 timeout: Duration,
11885 context: &str,
11886) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11887 let (mut e2e, mut timed_out_pids) = run_commands(
11888 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11889 )
11890 .await;
11891 for o in &e2e {
11892 state.event(
11893 "verify",
11894 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11895 );
11896 }
11897 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11901 if verify_retried {
11902 state.event(
11903 "verify",
11904 format!(
11905 "{context}: verify could not build/link, not a test result — retrying once \
11906 before concluding"
11907 ),
11908 );
11909 let retried = run_commands(
11910 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11911 )
11912 .await;
11913 e2e = retried.0;
11914 timed_out_pids.extend(retried.1);
11917 for o in &e2e {
11918 state.event(
11919 "verify",
11920 format!(
11921 "{context}: retry `{}` -> {}",
11922 o.command,
11923 e2e_outcome_label(o)
11924 ),
11925 );
11926 }
11927 }
11928 (e2e, verify_retried, timed_out_pids)
11929}
11930
11931#[allow(clippy::too_many_arguments)]
11947async fn run_commands(
11948 state: &mut RunState,
11949 node: &str,
11950 task: &str,
11951 attempt: usize,
11952 shell: &[String],
11953 commands: &[String],
11954 cwd: &Path,
11955 timeout: Duration,
11956) -> (Vec<CommandOutcome>, Vec<u32>) {
11957 if commands.is_empty() {
11958 return (Vec::new(), Vec::new());
11963 }
11964 let mut out = Vec::new();
11965 let mut timed_out_pids = Vec::new();
11966 let total = commands.len();
11967 for (idx, command) in commands.iter().enumerate() {
11968 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11969 if let Err(e) = state.save() {
11970 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11971 }
11972 let started = Instant::now();
11973 let mut cmd = tokio::process::Command::new(&shell[0]);
11974 cmd.quiet();
11975 for key in crate::worker::INVOCATION_ENV {
11979 cmd.env_remove(key);
11980 }
11981 cmd.args(&shell[1..])
11982 .arg(command)
11983 .current_dir(cwd)
11984 .stdin(std::process::Stdio::null())
11985 .stdout(std::process::Stdio::piped())
11986 .stderr(std::process::Stdio::piped())
11987 .kill_on_drop(true);
11988 let mut outputs = Vec::new();
11989 let protected = (|| -> Result<()> {
11990 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11991 outputs.push(guard);
11992 }
11993 if let Some(cache) = state.config.cache_dir() {
11994 outputs.push(crate::retention::OutputLease::cache(
11995 &crate::run::home(),
11996 &cache,
11997 false,
11998 )?);
11999 }
12000 Ok(())
12001 })();
12002 if let Err(e) = protected {
12003 out.push(CommandOutcome {
12004 command: command.clone(),
12005 code: None,
12006 output_tail: format!("output ownership unavailable: {e:#}"),
12007 duration_ms: 0,
12008 resource_blocked: true,
12009 });
12010 continue;
12011 }
12012 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12013 let (code, body) = match spawned {
12014 Ok(mut running) => {
12015 let pid = running.child().id();
12016 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12017 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12018 let mut paths = vec![cwd.to_owned()];
12019 if let Some(cache) = state.config.cache_dir() {
12020 paths.push(cache);
12021 }
12022 let mut pressure = None;
12023 let status = tokio::select! {
12024 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12025 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12026 pressure = Some(reason); None
12027 }
12028 };
12029 let (code, failure) = match status {
12030 Some(Ok(Ok(status))) => (
12031 status.code(),
12032 status
12033 .code()
12034 .is_none()
12035 .then(|| format!("command terminated: {status}")),
12036 ),
12037 Some(Ok(Err(error))) => {
12038 (None, Some(format!("failed waiting for command: {error}")))
12039 }
12040 Some(Err(_)) => {
12041 if !running.coordinated()
12045 && let Some(pid) = pid
12046 {
12047 timed_out_pids.push(pid);
12048 }
12049 (
12050 None,
12051 Some(format!("timed out after {}s", timeout.as_secs())),
12052 )
12053 }
12054 None => (None, None),
12055 };
12056 running.finish().await;
12057 if let Some(reason) = pressure {
12058 state.resource_block = Some(crate::run::ResourceBlock {
12059 cause: crate::run::ResourceBlockCause::Other,
12060 node: node.to_owned(),
12061 kind: "disk pressure".to_owned(),
12062 detail: reason.clone(),
12063 at: Timestamp::now(),
12064 });
12065 out.push(CommandOutcome {
12066 command: command.clone(),
12067 code: None,
12068 output_tail: reason,
12069 duration_ms: started.elapsed().as_millis() as u64,
12070 resource_blocked: true,
12071 });
12072 break;
12073 }
12074 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12075 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12076 if let Some(failure) = failure {
12077 body.push('\n');
12078 body.push_str(&failure);
12079 }
12080 (code, body)
12081 }
12082 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12083 };
12084 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12085 if !body.is_empty() {
12086 match crate::run::write_verification_artifact(state, task, &body) {
12090 Ok(name) if summary != body => {
12091 summary = format!("[full output: artifacts/{name}]\n{summary}");
12092 }
12093 Ok(_) => {}
12094 Err(e) => {
12095 let warning = format!("could not save the full {task} output: {e:#}");
12096 tracing::warn!("{warning}");
12097 state.event(node, warning);
12098 if summary != body {
12099 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12100 }
12101 }
12102 }
12103 }
12104 out.push(CommandOutcome {
12105 command: command.clone(),
12106 code,
12107 output_tail: summary,
12108 duration_ms: started.elapsed().as_millis() as u64,
12109 resource_blocked: false,
12110 });
12111 }
12112 state.task_finished(task);
12113 if let Err(e) = state.save() {
12114 tracing::warn!("could not persist the end of {task}: {e:#}");
12115 }
12116 (out, timed_out_pids)
12117}
12118
12119fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12131 let repo = repo.display();
12132 match style {
12133 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12134 MergeStyle::Squash => {
12135 let subject = message
12138 .lines()
12139 .next()
12140 .unwrap_or(branch)
12141 .replace(['\\', '"', '$', '`'], "");
12142 format!(
12143 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12144 )
12145 }
12146 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12147 }
12148}
12149
12150const PR_TITLE_MAX: usize = 240;
12161
12162fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12173 let line = queue::first_line(text)?;
12174 let chars: Vec<char> = line.chars().collect();
12175 let (kept, mark) = if chars.len() <= max {
12176 (line.to_owned(), "")
12177 } else if let Some(end) = sentence_end(&chars, max) {
12178 (chars[..end].iter().collect::<String>(), "")
12179 } else {
12180 let room = max.saturating_sub(3);
12181 let cut = if chars[room].is_whitespace() {
12184 room
12185 } else {
12186 chars[..room]
12187 .iter()
12188 .rposition(|c| c.is_whitespace())
12189 .unwrap_or(room)
12190 };
12191 let head: String = chars[..cut].iter().collect();
12192 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12193 (head.to_owned(), "...")
12194 };
12195 if kept.is_empty() {
12196 return None;
12197 }
12198 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12199 return None;
12200 }
12201 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12202 return None;
12203 }
12204 Some(format!("{kept}{mark}"))
12205}
12206
12207fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12211 const MIN: usize = 20;
12212 for i in MIN..max.min(chars.len()) {
12213 if !matches!(chars[i], '.' | '!' | '?') {
12214 continue;
12215 }
12216 let Some(&next) = chars.get(i + 1) else {
12217 continue;
12218 };
12219 if !next.is_whitespace() {
12220 continue;
12221 }
12222 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12223 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12224 continue;
12225 }
12226 let word: String = chars[..i]
12227 .iter()
12228 .rev()
12229 .take_while(|c| !c.is_whitespace())
12230 .collect::<Vec<_>>()
12231 .into_iter()
12232 .rev()
12233 .collect();
12234 let word = word.to_ascii_lowercase();
12235 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12236 continue;
12237 }
12238 let end = chars[..i]
12239 .iter()
12240 .rposition(|c| !c.is_whitespace())
12241 .map_or(i, |p| p + 1);
12242 return Some(end);
12243 }
12244 None
12245}
12246
12247struct PrMessage {
12252 title: String,
12253 body: String,
12254}
12255
12256impl PrMessage {
12257 fn commit_message(&self) -> String {
12261 format!("{}\n\n{}", self.title, self.body)
12262 }
12263}
12264
12265fn title_marker(line: &str) -> Option<&str> {
12267 let line = line.trim();
12268 let head = line.get(..6)?;
12269 head.eq_ignore_ascii_case("title:")
12270 .then(|| line[6..].trim())
12271}
12272
12273fn summary_title(summary: &str) -> Option<String> {
12278 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12279 let raw = title_marker(first)?;
12280 if raw.is_empty() {
12281 return None;
12282 }
12283 let title = queue::title_from(raw, PR_TITLE_MAX);
12284 let lower = title.to_ascii_lowercase();
12285 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12286 return None;
12287 }
12288 Some(title)
12289}
12290
12291const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12293
12294pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12296
12297pub(crate) fn is_review_run(state: &RunState) -> bool {
12301 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12302}
12303
12304fn review_title(state: &RunState) -> Option<String> {
12309 english_subject(
12310 state.reviewed_commits.as_ref()?.first()?,
12311 state.config.graph.github_english(),
12312 )
12313}
12314
12315fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12318 let raw = raw.trim();
12319 if raw.is_empty() {
12320 return None;
12321 }
12322 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12323 return None;
12324 }
12325 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12326 return None;
12327 }
12328 let title = queue::title_from(raw, PR_TITLE_MAX);
12329 let lower = title.to_ascii_lowercase();
12330 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12331 return None;
12332 }
12333 Some(title)
12334}
12335
12336#[derive(Debug, Clone, PartialEq, Eq)]
12339struct BranchFacts {
12340 commits: Vec<(String, String)>,
12342 stat: String,
12344}
12345
12346const STAT_FILE_LINES: usize = 25;
12348const COMMITS_MAX_CHARS: usize = 20_000;
12350
12351async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12354 let commits = git::commit_log(repo, base, branch).await.ok()?;
12355 if commits.is_empty() {
12356 return None;
12357 }
12358 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12359 let lines: Vec<&str> = stat.lines().collect();
12360 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12361 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12362 let more = format!(" ... {omitted} more file(s)");
12363 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12364 kept.push(&more);
12365 kept.push(lines[lines.len() - 1]);
12366 kept.join("\n")
12367 } else {
12368 lines.join("\n")
12369 };
12370 Some(BranchFacts { commits, stat })
12371}
12372
12373fn markdown_safe(text: &str) -> String {
12376 text.replace("</details>", "</details>")
12377 .replace("\x60\x60\x60", "~~~")
12378}
12379
12380fn neutral_title(state: &RunState, winner: char) -> String {
12381 format!(
12382 "chore: land candidate {} of run {}",
12383 winner.to_ascii_uppercase(),
12384 state.id
12385 )
12386}
12387
12388pub(crate) async fn repair_landing_gate(
12391 state: &mut RunState,
12392 winner: &Candidate,
12393 outcomes: &[CommandOutcome],
12394) -> Result<GateFix> {
12395 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12396 return Ok(GateFix::Stop);
12397 }
12398 let mut runner = Runner {
12399 roles: state.config.resolve_roles()?,
12400 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12401 state: state.clone(),
12402 pause: Pause::new(),
12403 interrupt: Pause::new(),
12404 };
12405 let result = runner.gate_fix_round(winner, outcomes).await;
12406 *state = runner.state;
12407 result
12408}
12409
12410pub(crate) async fn verify_current_head(
12413 state: &mut RunState,
12414 winner: &Candidate,
12415 wait: &crate::repo_lock::WaitPolicy,
12416) -> Result<bool> {
12417 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12418 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12420 state
12421 .verification_history
12422 .push(crate::run::VerificationEvidence {
12423 superseded_by: Some(head.clone()),
12424 archived_at: Some(Timestamp::now()),
12425 fixes_spent: Some(state.gate_fixes.len()),
12426 gate_head: None,
12427 gate_ran: false,
12428 gate: Vec::new(),
12429 e2e: state.head_e2e.take(),
12430 });
12431 }
12432 if let Some(e) = &state.head_e2e
12433 && e.head == head
12434 {
12435 if e.passed {
12436 return Ok(true);
12437 }
12438 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12439 state.status = RunStatus::Blocked;
12440 state.save()?;
12441 return Ok(false);
12442 }
12443 } else if state.reviews.last().is_some_and(|r| {
12444 r.verified_head.as_deref() == Some(head.as_str())
12445 && (r.e2e_status() == E2eStatus::Passed
12446 || (r.e2e_status() == E2eStatus::NotConfigured
12447 && state.config.verify.e2e.is_empty()))
12448 }) || state
12449 .gate_repair_e2e
12450 .as_ref()
12451 .is_some_and(|e| e.head == head && e.passed)
12452 {
12453 return Ok(true);
12454 }
12455 if let Some(e) = &state.gate_repair_e2e
12456 && e.head == head
12457 && !e.passed
12458 && !e.outcomes.is_empty()
12459 && !verify_inconclusive(&e.outcomes)
12460 {
12461 state.status = RunStatus::Blocked;
12462 state.save()?;
12463 return Ok(false);
12464 }
12465 state.head_e2e = Some(crate::run::GateRepairE2e {
12466 head: head.clone(),
12467 passed: false,
12468 outcomes: Vec::new(),
12469 });
12470 state.save()?;
12471 let commands = state.config.verify.e2e.clone();
12472 let shell = state.config.shell();
12473 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12474 let cache_dir = state.config.cache_dir();
12475 let outcomes = if commands.is_empty() {
12476 Vec::new()
12477 } else {
12478 let (outcomes, _) = with_cache_lease(
12479 state,
12480 cache_dir.as_deref(),
12481 "e2e",
12482 "e2e",
12483 &winner.worktree,
12484 &head,
12485 timeout,
12486 "current head e2e",
12487 wait,
12488 |state, budget| async move {
12489 run_e2e_with_retry(
12490 state,
12491 &shell,
12492 &commands,
12493 &winner.worktree,
12494 budget,
12495 "current head e2e",
12496 )
12497 .await
12498 },
12499 )
12500 .await?;
12501 outcomes
12502 };
12503 let unresolved = verify_inconclusive(&outcomes);
12504 let passed = !unresolved
12505 && outcomes.iter().all(CommandOutcome::ok)
12506 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12507 if !outcomes.is_empty() && !unresolved {
12508 state.record_base_progress(&head);
12509 }
12510 state.head_e2e = Some(crate::run::GateRepairE2e {
12511 head: head.clone(),
12512 passed,
12513 outcomes,
12514 });
12515 if !passed && !unresolved {
12516 state.status = RunStatus::Blocked;
12517 state.event(
12518 "e2e",
12519 format!("verify.e2e failed on head {head}; nothing published"),
12520 );
12521 }
12522 state.save()?;
12523 Ok(passed)
12524}
12525
12526pub(crate) async fn check_gate_repair_e2e(
12529 state: &mut RunState,
12530 winner: &Candidate,
12531 wait: &crate::repo_lock::WaitPolicy,
12532) -> Result<GateFix> {
12533 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12534 return Ok(GateFix::Retry);
12535 }
12536 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12537 if state
12538 .gate_repair_e2e
12539 .as_ref()
12540 .is_some_and(|r| r.head == head && r.passed)
12541 {
12542 return Ok(GateFix::Retry);
12543 }
12544 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12545 head: head.clone(),
12546 passed: false,
12547 outcomes: Vec::new(),
12548 });
12549 state.save()?;
12550 let commands = state.config.verify.e2e.clone();
12551 let outcomes = if commands.is_empty() {
12552 Vec::new()
12553 } else {
12554 let shell = state.config.shell();
12555 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12556 let cache_dir = state.config.cache_dir();
12557 let (outcomes, _) = with_cache_lease(
12558 state,
12559 cache_dir.as_deref(),
12560 "e2e",
12561 "e2e",
12562 &winner.worktree,
12563 &head,
12564 timeout,
12565 "gate repair e2e",
12566 wait,
12567 |state, budget| async move {
12568 run_e2e_with_retry(
12569 state,
12570 &shell,
12571 &commands,
12572 &winner.worktree,
12573 budget,
12574 "gate repair e2e",
12575 )
12576 .await
12577 },
12578 )
12579 .await?;
12580 outcomes
12581 };
12582 let deferred = verify_inconclusive(&outcomes);
12583 let passed = outcomes.iter().all(CommandOutcome::ok)
12584 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12585 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12586 head,
12587 passed,
12588 outcomes,
12589 });
12590 state.save()?;
12591 if deferred {
12592 Ok(GateFix::Defer)
12593 } else if !passed {
12594 state.event(
12595 "gate",
12596 "verify.e2e failed after gate repair; nothing published",
12597 );
12598 Ok(GateFix::Stop)
12599 } else {
12600 Ok(GateFix::Retry)
12601 }
12602}
12603
12604pub(crate) async fn run_landing_gate(
12606 state: &mut RunState,
12607 winner: &Candidate,
12608 wait: &crate::repo_lock::WaitPolicy,
12609) -> Result<Vec<CommandOutcome>> {
12610 let shell = state.config.shell();
12611 let gate_commands = state.config.verify.gate.clone();
12612 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12613 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12614 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12615 Vec::new()
12616 } else {
12617 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12618 let cache_dir = state.config.cache_dir();
12619 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12620 let (outcomes, _) = with_cache_lease(
12621 state,
12622 cache_dir.as_deref(),
12623 "gate",
12624 "gate",
12625 &winner.worktree,
12626 &head,
12627 timeout,
12628 "final gate",
12629 wait,
12630 |state, budget| {
12631 let shell = shell.clone();
12632 let gate_commands = gate_commands.clone();
12633 let winner = winner.clone();
12634 async move {
12635 Runner::run_pre_gate(state, &winner).await;
12636 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12638 let (outcomes, timed_out_pids) = run_commands(
12639 state,
12640 "gate",
12641 "gate",
12642 0,
12643 &shell,
12644 &gate_commands,
12645 &winner.worktree,
12646 budget,
12647 )
12648 .await;
12649 (outcomes, false, timed_out_pids)
12650 }
12651 },
12652 )
12653 .await?;
12654 outcomes
12655 };
12656 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12657 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12658 let mut outcomes = outcomes;
12659 outcomes.push(CommandOutcome {
12660 command: "gate head stability".into(), code: None,
12661 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12662 duration_ms: 0, resource_blocked: false,
12663 });
12664 return Ok(outcomes);
12665 }
12666 if outcomes.is_empty() {
12667 state.event("gate", "no gate commands configured; not tested");
12672 }
12673 for o in &outcomes {
12674 state.event(
12675 "gate",
12676 format!(
12677 "`{}` -> {}",
12678 o.command,
12679 if o.ok() {
12680 "pass".to_owned()
12681 } else {
12682 format!(
12683 "FAIL ({:?})\n{}",
12684 o.code,
12685 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12686 )
12687 }
12688 ),
12689 );
12690 }
12691 Ok(outcomes)
12692}
12693
12694pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12700 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12701 ""
12702 } else {
12703 pr_title
12704 }
12705}
12706
12707pub fn landing_subject_source(state: &RunState) -> String {
12711 if is_review_run(state) {
12712 let winner = state.candidates.first().map_or('A', |c| c.label);
12713 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12714 }
12715 state.instruction.clone()
12716}
12717
12718fn summary_without_title(summary: &str) -> String {
12721 let mut lines = summary.trim().lines().peekable();
12722 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12723 lines.next();
12724 }
12725 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12726}
12727
12728#[cfg(test)]
12740fn pr_message(state: &RunState, winner: char) -> PrMessage {
12741 pr_message_with(state, winner, None)
12742}
12743
12744#[cfg(test)]
12747fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12748 let raw = pr_message_raw(
12749 state,
12750 winner,
12751 facts,
12752 crate::pr_outcome::FinalHead::Unreported,
12753 ChangeNote::NotAsked,
12754 );
12755 let id = crate::scrub::Identity::current();
12756 PrMessage {
12757 title: crate::scrub::scrub(&raw.title, &id),
12758 body: crate::scrub::scrub(&raw.body, &id),
12759 }
12760}
12761
12762#[derive(Debug, Clone, Copy)]
12764enum ChangeNote<'a> {
12765 NotAsked,
12767 Summary(&'a crate::run::ChangeSummary),
12769 Unavailable(&'a str),
12771}
12772
12773fn pr_message_raw(
12774 state: &RunState,
12775 winner: char,
12776 facts: Option<&BranchFacts>,
12777 head: crate::pr_outcome::FinalHead<'_>,
12778 change: ChangeNote<'_>,
12779) -> PrMessage {
12780 if !matches!(change, ChangeNote::NotAsked) {
12781 let (title, mut body) = match change {
12782 ChangeNote::Summary(c) => (
12783 c.title.clone(),
12784 format!("## Summary\n\n{}\n\n", c.body.trim()),
12785 ),
12786 ChangeNote::Unavailable(why) => (
12787 neutral_title(state, winner),
12788 format!(
12789 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12790 ),
12791 ),
12792 ChangeNote::NotAsked => unreachable!(),
12793 };
12794 body.push_str(&public_pr_evidence(state, head));
12795 body.push_str(&format!(
12796 "\n---\nmagi:run/{} magi:candidate-{}\n",
12797 state.id,
12798 winner.to_ascii_lowercase()
12799 ));
12800 return PrMessage { title, body };
12801 }
12802 pr_preview(state, winner, facts, head)
12803}
12804
12805fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12808 let mut body = crate::pr_outcome::review_rounds(state);
12809 body.push_str(&crate::pr_outcome::final_verification(state, head));
12810 let open = state.last_round_findings();
12811 if !open.is_empty() {
12812 body.push_str("## Open review findings\n\n");
12813 for f in open {
12814 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12815 }
12816 body.push('\n');
12817 }
12818 if let Some(c) = state.change_summary.as_ref()
12819 && head == crate::pr_outcome::FinalHead::At(&c.head)
12820 {
12821 body.push_str(&format!(
12822 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12823 c.base, c.head
12824 ));
12825 }
12826 body
12827}
12828
12829fn pr_preview(
12831 state: &RunState,
12832 winner: char,
12833 facts: Option<&BranchFacts>,
12834 head: crate::pr_outcome::FinalHead<'_>,
12835) -> PrMessage {
12836 let summary = state
12837 .candidates
12838 .iter()
12839 .find(|c| c.label == winner)
12840 .map(|c| c.summary.as_str())
12841 .unwrap_or_default();
12842 let review = is_review_run(state);
12845 let title = if review {
12846 facts
12847 .and_then(|f| {
12848 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12849 })
12850 .or_else(|| review_title(state))
12851 .or_else(|| {
12852 state
12853 .candidates
12854 .iter()
12855 .find(|c| c.label == winner)
12856 .filter(|c| !c.branch.starts_with("magi/"))
12857 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12858 })
12859 .unwrap_or_else(|| neutral_title(state, winner))
12860 } else {
12861 summary_title(summary).unwrap_or_else(|| {
12862 english_title(
12863 &state.instruction,
12864 PR_TITLE_MAX,
12865 state.config.graph.github_english(),
12866 )
12867 .unwrap_or_else(|| neutral_title(state, winner))
12868 })
12869 };
12870
12871 let mut body = String::new();
12872 let what = summary_without_title(summary);
12873 if !what.is_empty() {
12874 body.push_str("## Summary\n\n");
12875 body.push_str(&what);
12876 body.push_str("\n\n");
12877 }
12878
12879 body.push_str(&crate::pr_outcome::review_rounds(state));
12881 body.push_str(&crate::pr_outcome::final_verification(state, head));
12882 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12883
12884 let open = state.open_findings();
12885 if !open.is_empty() {
12886 body.push_str("## Open review findings\n\n");
12887 for f in &open {
12888 body.push_str(&format!(
12889 "- `{}` [{:?}] {}\n",
12890 f.id, f.severity, "Context retained in local review records."
12891 ));
12892 }
12893 body.push('\n');
12894 }
12895
12896 if let Some(fix) = fix
12897 && !fix.rejected.is_empty()
12898 {
12899 body.push_str("## Declined by the fixer\n\n");
12900 for r in &fix.rejected {
12901 body.push_str(&format!(
12902 "- `{}`: {}\n",
12903 r.id, "Reason retained in local review records."
12904 ));
12905 }
12906 body.push('\n');
12907 }
12908
12909 if review {
12910 body.push_str("## Commits under review\n\n");
12912 if let Some(facts) = facts {
12913 let mut left = COMMITS_MAX_CHARS;
12914 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12915 let mut entry = format!("- {}\n", markdown_safe(subject));
12916 for l in markdown_safe(text).lines() {
12917 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12918 }
12919 if left == 0 {
12920 body.push_str(&format!(
12921 "- ... {} more commit(s)\n",
12922 facts.commits.len() - i
12923 ));
12924 break;
12925 }
12926 if entry.len() > left {
12927 let mut end = left;
12930 while !entry.is_char_boundary(end) {
12931 end -= 1;
12932 }
12933 entry.truncate(end);
12934 entry.push_str("\n ... (truncated)\n");
12935 left = 0;
12936 } else {
12937 left -= entry.len();
12938 }
12939 body.push_str(&entry);
12940 }
12941 if !facts.stat.trim().is_empty() {
12942 body.push_str(&format!(
12943 "\n## Diff stat\n\n```\n{}\n```\n",
12944 markdown_safe(facts.stat.trim())
12945 ));
12946 }
12947 } else {
12948 match &state.reviewed_commits {
12949 Some(subjects) => {
12950 for s in subjects {
12951 body.push_str(&format!("- {}\n", s.trim()));
12952 }
12953 }
12954 None => {
12955 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12958 body.push_str(rest.trim());
12959 body.push('\n');
12960 }
12961 }
12962 }
12963 } else {
12964 let task = state.instruction.trim();
12965 let task = if task.is_empty() {
12966 "(empty task)"
12967 } else {
12968 task
12969 };
12970 body.push_str(&format!(
12971 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12972 task.replace("</details>", "</details>")
12973 ));
12974 }
12975
12976 body.push_str(&format!(
12977 "\n---\nmagi:run/{} magi:candidate-{}\n",
12978 state.id,
12979 winner.to_ascii_lowercase()
12980 ));
12981
12982 PrMessage { title, body }
12983}
12984
12985fn seeded_instruction(state: &RunState) -> String {
12988 let mut inputs = Vec::new();
12989 if let Some(start) = &state.sources.start_from {
12990 inputs.push(format!(
12991 "Start from `{}` at `{}` (including ancestry).",
12992 start.token, start.sha
12993 ));
12994 }
12995 for pick in &state.sources.pick_commits {
12996 inputs.push(format!(
12997 "Pick `{}` at `{}` (single commit).",
12998 pick.token, pick.sha
12999 ));
13000 }
13001 if inputs.is_empty() {
13002 state.instruction.clone()
13003 } else {
13004 format!(
13005 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13006 state.instruction,
13007 inputs.join("\n")
13008 )
13009 }
13010}
13011
13012async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13016 let base = &state.base_branch;
13017 let mut against = base.clone();
13018 if mode == MergeMode::Pr {
13019 let remote = &state.config.merge.remote;
13023 let tracking = format!("{remote}/{base}");
13024 let fetched = git::fetch(repo, remote, base).await;
13025 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13026 against = tracking;
13027 } else {
13028 return false;
13029 }
13030 }
13031 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13032}
13033
13034fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13037 let mut detail = format!(
13038 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13039 nothing to open a pull request for"
13040 );
13041 detail.push_str(
13042 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13043 );
13044 detail
13045}
13046
13047#[derive(Debug, PartialEq, Eq)]
13050enum PrPlan {
13051 Create,
13052 Adopt { url: String, title: String },
13053 Stop(String),
13054}
13055
13056fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13059 match found {
13060 Ok(land::OpenPr::None) => PrPlan::Create,
13061 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13062 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13063 "several open pull requests exist for this branch, not picking one: {}",
13064 urls.join(" ")
13065 )),
13066 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13067 }
13068}
13069
13070pub(crate) async fn gh_pr_create(
13072 cwd: &Path,
13073 base: &str,
13074 head: &str,
13075 title: &str,
13076 body: &str,
13077) -> Result<String> {
13078 let out = tokio::process::Command::new("gh")
13079 .args([
13080 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13081 ])
13082 .current_dir(cwd)
13083 .quiet()
13084 .stdin(std::process::Stdio::null())
13085 .output()
13086 .await
13087 .context("spawn gh")?;
13088 if out.status.success() {
13089 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13090 } else {
13091 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13092 }
13093}
13094
13095async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13097 match std::fs::symlink_metadata(path) {
13098 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13099 git::worktree_remove_clean(repo, path).await?;
13102 return Ok(());
13103 }
13104 Err(e) => return Err(e.into()),
13105 Ok(_) => {}
13106 }
13107 if !git::is_clean(path).await? {
13108 bail!(
13109 "worktree has uncommitted work; refusing removal: {}",
13110 path.display()
13111 );
13112 }
13113 if !git::worktree_remove_clean(repo, path).await? {
13114 bail!("worktree removal refused: {}", path.display());
13115 }
13116 Ok(())
13117}
13118
13119pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13128 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13129 bail!("refusing fold: {reason}");
13130 }
13131 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13132 bail!("candidate preservation incomplete; refusing cleanup");
13133 }
13134 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13138 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13139 let _driver = crate::driver::maintenance(home, &state.id)?;
13140 let mut output_guards = Vec::new();
13141 for entry in std::fs::read_dir(state.worktree_root())
13142 .into_iter()
13143 .flatten()
13144 .flatten()
13145 {
13146 if entry.path().join(".git").is_file() {
13147 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13148 }
13149 }
13150 let repo = state.repo.clone();
13151 let wait = repository_wait(state, None);
13154 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13155 Ok(lease) => lease,
13156 Err(e) => {
13157 state.save_under(home)?;
13158 return Err(e);
13159 }
13160 };
13161 let _source_outputs = if state.status == RunStatus::Merged
13164 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13165 {
13166 verify_merged_fold_sources(state, drop_winner, home).await?
13167 } else {
13168 Vec::new()
13169 };
13170 let root = state.worktree_root();
13171 let winner = state.tally.as_ref().map(|t| t.winner);
13172 let mut removed = Vec::new();
13173
13174 for i in 0..state.candidates.len() {
13175 let c = state.candidates[i].clone();
13176 let is_winner = Some(c.label) == winner;
13177 if is_winner && !drop_winner {
13178 continue;
13179 }
13180 let existed = c.worktree.exists();
13181 remove_preserved_worktree(&repo, &c.worktree).await?;
13184 if existed {
13185 removed.push(c.worktree.to_string_lossy().into_owned());
13186 }
13187 let handed_over = state.released_branches.contains(&c.branch);
13190 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13191 if !git::branch_delete(&repo, &c.branch).await? {
13192 bail!("candidate branch removal refused");
13193 }
13194 removed.push(c.branch.clone());
13195 }
13196 state.candidates[i].folded = true;
13197 }
13198
13199 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13200 let path = name.path();
13201 let keep = !drop_winner
13202 && winner.is_some_and(|w| {
13203 path.file_name()
13204 .is_some_and(|n| n == format!("cand-{w}").as_str())
13205 });
13206 if keep {
13207 continue;
13208 }
13209 remove_preserved_worktree(&repo, &path).await?;
13210 removed.push(path.to_string_lossy().into_owned());
13211 }
13212
13213 remove_if_empty(&root);
13222
13223 if state.enabled_worktree_config && drop_winner {
13224 git::release_worktree_config(&repo).await.ok();
13228 state.enabled_worktree_config = false;
13229 }
13230 state.save_under(home)?;
13231 Ok(removed)
13232}
13233
13234async fn verify_merged_fold_sources(
13235 state: &RunState,
13236 drop_winner: bool,
13237 home: &Path,
13238) -> Result<Vec<std::fs::File>> {
13239 let winner = state.tally.as_ref().map(|t| t.winner);
13240 let mut heads = BTreeSet::new();
13241 for candidate in &state.candidates {
13242 if Some(candidate.label) == winner && !drop_winner {
13243 continue;
13244 }
13245 anyhow::ensure!(
13246 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13247 "candidate recovery source is outside the inventoried run root: {}",
13248 candidate.worktree.display()
13249 );
13250 match std::fs::symlink_metadata(&candidate.worktree) {
13251 Ok(_) => {}
13252 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13253 Err(error) => return Err(error.into()),
13254 }
13255 if !state.released_branches.contains(&candidate.branch)
13256 && git::branch_exists(&state.repo, &candidate.branch).await?
13257 {
13258 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13259 }
13260 }
13261 let mut cargo_guards = Vec::new();
13262 let entries = match std::fs::read_dir(state.worktree_root()) {
13263 Ok(entries) => Some(entries),
13264 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13265 Err(error) => return Err(error.into()),
13266 };
13267 for entry in entries.into_iter().flatten() {
13268 let path = entry?.path();
13269 let metadata = std::fs::symlink_metadata(&path)?;
13270 anyhow::ensure!(
13271 metadata.is_dir() && !metadata.file_type().is_symlink(),
13272 "non-directory or symlink recovery source retained: {}",
13273 path.display()
13274 );
13275 if !drop_winner
13276 && winner.is_some_and(|w| {
13277 path.file_name()
13278 .is_some_and(|name| name == format!("cand-{w}").as_str())
13279 })
13280 {
13281 continue;
13282 }
13283 anyhow::ensure!(
13284 path.join(".git").is_file(),
13285 "unregistered recovery source retained: {}",
13286 path.display()
13287 );
13288 anyhow::ensure!(
13289 git::git(
13290 &path,
13291 &[
13292 "status",
13293 "--porcelain",
13294 "--untracked-files=all",
13295 "--ignored",
13296 "--",
13297 ".",
13298 ":(exclude)target"
13299 ]
13300 )
13301 .await?
13302 .is_empty(),
13303 "source changes or ignored recovery files retained: {}",
13304 path.display()
13305 );
13306 match std::fs::symlink_metadata(path.join("target")) {
13307 Ok(_) => {
13308 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13309 }
13310 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13311 Err(error) => return Err(error.into()),
13312 }
13313 for checkpoint in [
13314 "rebase-merge",
13315 "rebase-apply",
13316 "sequencer",
13317 "CHERRY_PICK_HEAD",
13318 "MERGE_HEAD",
13319 "REBASE_HEAD",
13320 "REVERT_HEAD",
13321 "BISECT_LOG",
13322 "AUTO_MERGE",
13323 "MERGE_MSG",
13324 ] {
13325 let location = git::git(
13326 &path,
13327 &[
13328 "rev-parse",
13329 "--path-format=absolute",
13330 "--git-path",
13331 checkpoint,
13332 ],
13333 )
13334 .await?;
13335 anyhow::ensure!(
13336 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13337 if e.kind() == std::io::ErrorKind::NotFound),
13338 "Git recovery checkpoint retained: {checkpoint}"
13339 );
13340 }
13341 heads.insert(git::rev_parse(&path, "HEAD").await?);
13342 }
13343 crate::fold_evidence::preserve(state, home, &heads).await?;
13344
13345 Ok(cargo_guards)
13346}
13347
13348fn remove_if_empty(dir: &Path) {
13359 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13360 std::fs::remove_dir(dir).ok();
13361 }
13362}
13363
13364pub fn worst_open(state: &RunState) -> Option<Severity> {
13366 state
13367 .reviews
13368 .last()?
13369 .reviews
13370 .iter()
13371 .flat_map(|r| r.findings.iter())
13372 .map(|f| f.severity)
13373 .max()
13374}
13375
13376async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13380 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13381 git::acquire_worktree_config(repo).await?;
13382 state.enabled_worktree_config = true;
13383 }
13384 Ok(())
13385}
13386
13387#[cfg(test)]
13388mod tests {
13389 #[tokio::test]
13390 async fn launcher_base_fetch_requires_remote_success() {
13391 let tmp = tempfile::tempdir().unwrap();
13392 let repo = tmp.path().join("repo");
13393 let remote = tmp.path().join("remote");
13394 std::fs::create_dir(&repo).unwrap();
13395 std::fs::create_dir(&remote).unwrap();
13396 git::git(&remote, &["init", "--bare", "-b", "main"])
13397 .await
13398 .unwrap();
13399 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13400 git::git(
13401 &repo,
13402 &[
13403 "-c",
13404 "user.name=Fixture",
13405 "-c",
13406 "user.email=fixture@example.invalid",
13407 "commit",
13408 "--allow-empty",
13409 "-m",
13410 "fixture",
13411 ],
13412 )
13413 .await
13414 .unwrap();
13415 git::git(
13416 &repo,
13417 &["remote", "add", "origin", remote.to_str().unwrap()],
13418 )
13419 .await
13420 .unwrap();
13421 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13422 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13423 assert_eq!(
13424 resolve_base(&repo, "main", "origin", None)
13425 .await
13426 .unwrap()
13427 .commit,
13428 head
13429 );
13430 #[cfg(unix)]
13431 {
13432 let policy: crate::config::Coordination = toml::from_str(
13433 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13434 ).unwrap();
13435 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13436 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13437 let held = crate::repo_lock::SeatLease::operation_with(
13438 &repo,
13439 Some(&policy),
13440 &crate::repo_lock::WaitPolicy::default(),
13441 )
13442 .await
13443 .unwrap();
13444 assert!(held.is_some());
13445 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13446 tokio::pin!(fetch);
13447 assert!(
13448 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13449 .await
13450 .is_err()
13451 );
13452 assert!(
13453 !repo.join(".git/FETCH_HEAD").exists(),
13454 "no fetch before admission"
13455 );
13456 drop(held);
13457 let fetched = fetch.await.unwrap();
13458 assert_eq!(fetched.commit, head);
13459 assert_eq!(fetched.remote, "origin");
13460 assert_eq!(fetched.branch, "main");
13461 }
13462 git::git(
13464 &repo,
13465 &[
13466 "remote",
13467 "set-url",
13468 "origin",
13469 tmp.path().join("missing").to_str().unwrap(),
13470 ],
13471 )
13472 .await
13473 .unwrap();
13474 let error = resolve_base(&repo, "main", "origin", None)
13475 .await
13476 .unwrap_err()
13477 .to_string();
13478 assert!(error.contains("cannot read origin/main"));
13479 assert!(error.contains("refusing to branch off"));
13480 }
13481
13482 #[test]
13483 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13484 let leaked = vec!["chore(deps): update a crate".to_owned()];
13485 let own = "fix(daemon): apply a chosen action";
13486 assert!(should_retitle("", own, &leaked));
13487 assert!(should_retitle(
13488 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13489 own,
13490 &leaked
13491 ));
13492 assert!(should_retitle(
13493 "chore: land candidate A of run 1",
13494 own,
13495 &leaked
13496 ));
13497 assert!(should_retitle(
13498 "magi: candidate A (uncommitted work)",
13499 own,
13500 &leaked
13501 ));
13502 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13503 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13504 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13505 }
13506
13507 #[test]
13508 fn pr_merge_plan_creates_adopts_or_stops() {
13509 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13510 assert_eq!(
13511 pr_merge_plan(Ok(land::OpenPr::One {
13512 url: "u".into(),
13513 title: "t".into()
13514 })),
13515 PrPlan::Adopt {
13516 url: "u".into(),
13517 title: "t".into()
13518 }
13519 );
13520 let PrPlan::Stop(many) =
13521 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13522 else {
13523 panic!("many must stop");
13524 };
13525 assert!(many.contains('a') && many.contains('b'));
13526 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13527 panic!("a failed lookup must stop");
13528 };
13529 assert!(err.contains("bad token"));
13530 }
13531
13532 use super::*;
13533 use crate::run::GateStatus;
13534 use std::collections::BTreeMap;
13535 use std::time::Duration;
13536
13537 fn conductor() -> AgentSpec {
13538 AgentSpec {
13539 id: "conductor".to_owned(),
13540 kind: crate::config::AgentKind::Command,
13541 model: None,
13542 command: vec!["true".to_owned()],
13543 extra_args: Vec::new(),
13544 env: BTreeMap::new(),
13545 prompt_delivery: None,
13546 max_turn_secs: None,
13547 }
13548 }
13549
13550 fn spec(id: &str) -> AgentSpec {
13551 AgentSpec {
13552 id: id.to_owned(),
13553 kind: crate::config::AgentKind::Command,
13554 model: None,
13555 command: vec!["true".to_owned()],
13556 extra_args: Vec::new(),
13557 env: BTreeMap::new(),
13558 prompt_delivery: None,
13559 max_turn_secs: None,
13560 }
13561 }
13562
13563 fn ids(xs: &[&str]) -> BTreeSet<String> {
13564 xs.iter().map(|s| (*s).to_owned()).collect()
13565 }
13566
13567 #[test]
13568 fn pick_successor_skips_an_agent_another_seat_holds() {
13569 let roster = [spec("a"), spec("b"), spec("c")];
13571 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13572 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13573 }
13574
13575 #[test]
13576 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13577 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13579 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13580 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13581 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13582 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13583 }
13584
13585 #[test]
13586 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13587 let roster = [spec("a"), spec("b")];
13588 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13589 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13590 let carried = ids(&["b"]);
13591 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13592 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13593 }
13594
13595 #[test]
13596 fn pick_successor_returns_none_without_any_untried_successor() {
13597 let roster = [spec("a"), spec("b")];
13598 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13599 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13600 let carried = ids(&["a"]);
13601 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13602 }
13603
13604 #[test]
13605 fn pick_successor_rescue_avoids_another_seats_occupant() {
13606 let roster = [spec("a"), spec("b"), spec("c")];
13608 let carried = ids(&["b", "c"]);
13609 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13610 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13611 }
13612
13613 #[test]
13614 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13615 let roster = [spec("a"), spec("b"), spec("c")];
13616 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13617 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13618 }
13619
13620 #[test]
13621 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13622 let roster = [spec("a"), spec("b"), spec("c")];
13623 let failed = ids(&["a", "b", "c"]);
13624 let mut tried = ids(&["b"]);
13626 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13627 assert_eq!(first.id, "a");
13628 tried.insert(first.id.clone());
13629 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13630 assert_eq!(second.id, "c");
13631 tried.insert(second.id.clone());
13632 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13633 }
13634
13635 #[test]
13636 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13637 let roster = [spec("a"), spec("b")];
13638 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13639 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13640 }
13641
13642 #[test]
13643 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13644 let roster = [spec("a"), spec("b"), spec("c")];
13645 let h = SeatHistory {
13646 failed: ids(&["a"]),
13647 last_ok: Some("b".to_owned()),
13648 last_fail: None,
13649 };
13650 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13651 let gone = SeatHistory {
13653 last_ok: Some("zzz".to_owned()),
13654 ..h.clone()
13655 };
13656 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13657 let failed = SeatHistory {
13658 failed: ids(&["a", "b"]),
13659 last_ok: Some("b".to_owned()),
13660 last_fail: None,
13661 };
13662 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13663 }
13664
13665 #[test]
13666 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13667 let first = SeatState::new("review-1", "alpha", 7);
13668 let next = handover_seat("review-1", "gamma", 7);
13669 assert_ne!(first.claude_session, next.claude_session);
13670 }
13671
13672 #[test]
13673 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13674 let mut state = state_with_summary("x", "y");
13675 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13676 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13677 assert_ne!(a.claude_session, b.claude_session);
13678 }
13679
13680 #[test]
13681 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13682 let roster = [spec("a"), spec("b")];
13683 let h = SeatHistory {
13684 failed: ids(&["a", "b"]),
13685 ..SeatHistory::default()
13686 };
13687 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13688 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13689 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13690 }
13691
13692 #[test]
13699 fn failure_signature_ignores_numbers_and_paths() {
13700 assert_eq!(
13701 failure_signature("exited with Some(2) and no usable output"),
13702 failure_signature("exited with Some(137) and no usable output")
13703 );
13704 assert_eq!(
13705 failure_signature("process exited with no usable output (exit code 2)"),
13706 failure_signature("process exited with no usable output (exit code 137)")
13707 );
13708 for (old, new) in [
13709 (
13710 "exited with Some(2) and no usable output",
13711 "process exited with no usable output (exit code 2)",
13712 ),
13713 (
13714 "exited with None and no usable output",
13715 "process exited with no usable output (exit code unavailable)",
13716 ),
13717 ] {
13718 let previous = FailClass::Other(failure_signature(old));
13719 let current = FailClass::Other(failure_signature(new));
13720 assert_eq!(previous, current, "resumption retains the failure class");
13721 assert!(!should_hand_over(Some(&previous), ¤t));
13722 }
13723 assert_eq!(
13724 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13725 failure_signature("cannot open /var/x.txt: denied")
13726 );
13727 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13728 }
13729
13730 #[test]
13731 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13732 let other = FailClass::Other("x".into());
13733 assert!(should_hand_over(None, &FailClass::Quota));
13734 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13735 assert!(should_hand_over(
13736 Some(&FailClass::Timeout),
13737 &FailClass::Timeout
13738 ));
13739 assert!(should_hand_over(None, &other));
13740 assert!(!should_hand_over(Some(&other), &other));
13741 assert!(should_hand_over(
13742 Some(&other),
13743 &FailClass::Other("y".into())
13744 ));
13745 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13747 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13748 }
13749
13750 #[test]
13751 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13752 let a = SeatState::new("judge-1", "alpha", 7);
13753 let b = handover_seat("judge-1", "beta", 7);
13754 assert_ne!(a.claude_session, b.claude_session);
13755 assert_eq!(b.turns, 0);
13756 }
13757
13758 #[test]
13759 fn a_timeout_is_classified_apart_from_other_failures() {
13760 assert_eq!(
13761 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13762 Some(FailClass::Timeout)
13763 );
13764 assert!(matches!(
13765 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13766 Some(FailClass::Other(_))
13767 ));
13768 }
13769
13770 #[test]
13771 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13772 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13773 let tried = BTreeSet::from(["beta".to_owned()]);
13774 let next = next_untried_in_roster(&roster, 1, &tried);
13777 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13778 }
13779
13780 #[test]
13781 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13782 let roster = vec![spec("alpha"), spec("beta")];
13783 let tried = BTreeSet::from(["beta".to_owned()]);
13784 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13788 }
13789
13790 #[test]
13791 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13792 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13793 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13794 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13798 }
13799
13800 #[test]
13801 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13802 let roster = vec![spec("a"), spec("a"), spec("b")];
13803 let tried = BTreeSet::from(["a".to_owned()]);
13804 let next = next_untried_in_roster(&roster, 0, &tried);
13805 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13806 }
13807
13808 #[test]
13809 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13810 let roster = vec![spec("a"), spec("b")];
13811 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13812 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13813 }
13814
13815 #[test]
13816 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13817 let dir = tempfile::tempdir().unwrap();
13818 let bay = dir.path().join("ffff");
13819
13820 remove_if_empty(&bay);
13822 assert!(!bay.exists());
13823
13824 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13827 remove_if_empty(&bay);
13828 assert!(bay.exists(), "non-empty directory must survive");
13829
13830 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13832 remove_if_empty(&bay);
13833 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13834 }
13835
13836 #[test]
13845 fn a_full_panel_that_found_nothing_is_clean() {
13846 assert!(round_is_clean(
13847 0,
13848 true,
13849 2,
13850 2,
13851 0,
13852 IncompleteReviewPolicy::Block
13853 ));
13854 }
13855
13856 #[test]
13857 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13858 assert!(!round_is_clean(
13859 0,
13860 true,
13861 1,
13862 2,
13863 0,
13864 IncompleteReviewPolicy::Block
13865 ));
13866 }
13867
13868 #[test]
13869 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13870 assert!(!round_is_clean(
13871 1,
13872 true,
13873 1,
13874 2,
13875 0,
13876 IncompleteReviewPolicy::Warn
13877 ));
13878 }
13879
13880 #[test]
13881 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13882 assert!(round_is_clean(
13883 0,
13884 true,
13885 1,
13886 2,
13887 0,
13888 IncompleteReviewPolicy::Warn
13889 ));
13890 }
13891
13892 #[test]
13893 fn a_full_panel_with_an_open_finding_is_not_clean() {
13894 assert!(!round_is_clean(
13895 1,
13896 true,
13897 2,
13898 2,
13899 0,
13900 IncompleteReviewPolicy::Block
13901 ));
13902 }
13903
13904 #[test]
13905 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13906 assert!(!round_is_clean(
13907 0,
13908 false,
13909 2,
13910 2,
13911 0,
13912 IncompleteReviewPolicy::Block
13913 ));
13914 }
13915
13916 #[test]
13923 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13924 assert!(round_is_clean(
13927 0,
13928 true,
13929 1,
13930 2,
13931 1,
13932 IncompleteReviewPolicy::Block
13933 ));
13934 }
13935
13936 #[test]
13937 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13938 assert!(!round_is_clean(
13941 0,
13942 true,
13943 1,
13944 2,
13945 0,
13946 IncompleteReviewPolicy::Block
13947 ));
13948 }
13949
13950 #[test]
13951 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13952 assert!(!round_is_clean(
13953 1,
13954 true,
13955 1,
13956 2,
13957 1,
13958 IncompleteReviewPolicy::Block
13959 ));
13960 assert!(!round_is_clean(
13961 0,
13962 false,
13963 1,
13964 2,
13965 1,
13966 IncompleteReviewPolicy::Block
13967 ));
13968 }
13969
13970 #[test]
13971 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13972 assert!(!round_is_clean(
13976 0,
13977 true,
13978 0,
13979 2,
13980 2,
13981 IncompleteReviewPolicy::Block
13982 ));
13983 }
13984
13985 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13986 CommandOutcome {
13987 command: "test".to_owned(),
13988 code,
13989 output_tail: String::new(),
13990 duration_ms: 0,
13991 resource_blocked,
13992 }
13993 }
13994
13995 #[test]
13996 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13997 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13998 assert!(
13999 !verify_inconclusive(&[outcome(Some(1), false)]),
14000 "an ordinary failure is still evidence about the patch"
14001 );
14002 assert!(verify_inconclusive(&[outcome(None, true)]));
14003 assert!(
14004 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14005 "one inconclusive outcome taints the whole batch"
14006 );
14007 assert!(!verify_inconclusive(&[]));
14008 }
14009
14010 #[tokio::test]
14011 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14012 let calls = std::sync::atomic::AtomicUsize::new(0);
14016 let started = Instant::now();
14017 wait_for_pids_with(
14018 &[123],
14019 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14020 Duration::from_millis(5),
14021 Duration::from_secs(5),
14022 )
14023 .await;
14024 assert!(
14025 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14026 "must keep checking rather than deciding on the first answer"
14027 );
14028 assert!(
14029 started.elapsed() < Duration::from_secs(1),
14030 "must return the moment it is confirmed dead, not wait out the ceiling"
14031 );
14032 }
14033
14034 #[tokio::test]
14035 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14036 let started = Instant::now();
14037 wait_for_pids_with(
14038 &[123],
14039 |_| true, Duration::from_millis(5),
14041 Duration::from_millis(30),
14042 )
14043 .await;
14044 let elapsed = started.elapsed();
14045 assert!(
14046 elapsed >= Duration::from_millis(30),
14047 "must not give up before its own ceiling: {elapsed:?}"
14048 );
14049 assert!(
14050 elapsed < Duration::from_secs(1),
14051 "must not wait past its own ceiling either: {elapsed:?}"
14052 );
14053 }
14054
14055 #[tokio::test]
14056 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14057 let started = Instant::now();
14058 wait_for_pids_with(
14059 &[],
14060 |_| true,
14061 Duration::from_secs(5),
14062 Duration::from_secs(5),
14063 )
14064 .await;
14065 assert!(
14066 started.elapsed() < Duration::from_millis(200),
14067 "an empty pid list has nothing to confirm"
14068 );
14069 }
14070
14071 fn review_round(
14077 clean: bool,
14078 blocking: usize,
14079 answered: usize,
14080 expected: usize,
14081 progressed: bool,
14082 e2e_ok: bool,
14083 ) -> ReviewRound {
14084 ReviewRound {
14085 round: 1,
14086 head: "h".to_owned(),
14087 verified_head: None,
14088 verified_at: None,
14089 reviews: Vec::new(),
14090 e2e: vec![CommandOutcome {
14091 command: "test".to_owned(),
14092 code: Some(if e2e_ok { 0 } else { 1 }),
14093 output_tail: String::new(),
14094 duration_ms: 0,
14095 resource_blocked: false,
14096 }],
14097 verify_retried: false,
14098 e2e_deferred: false,
14099 e2e_defer_reason: None,
14100 fix: None,
14101 blocking,
14102 answered,
14103 expected,
14104 clean,
14105 progressed,
14106 vote_split: false,
14107 reconsideration: Vec::new(),
14108 verdict: None,
14109 }
14110 }
14111
14112 #[test]
14113 fn review_conclusion_is_none_when_nothing_has_run() {
14114 assert_eq!(review_conclusion(&[], 3), None);
14115 }
14116
14117 #[test]
14118 fn review_conclusion_is_none_while_rounds_remain() {
14119 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14120 assert_eq!(review_conclusion(&rounds, 3), None);
14121 }
14122
14123 #[test]
14124 fn review_conclusion_is_gating_once_a_round_is_clean() {
14125 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14126 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14127 }
14128
14129 #[test]
14130 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14131 let rounds = vec![
14132 review_round(false, 1, 2, 2, true, true),
14133 review_round(false, 1, 2, 2, true, true),
14134 ];
14135 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14136 }
14137
14138 #[test]
14139 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14140 let rounds = vec![
14141 review_round(false, 1, 2, 2, true, true),
14142 review_round(false, 1, 2, 2, true, false),
14143 ];
14144 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14145 }
14146
14147 #[test]
14148 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14149 let mut blocked = review_round(false, 1, 2, 2, true, false);
14156 blocked.e2e[0].resource_blocked = true;
14157 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14158 assert_eq!(review_conclusion(&rounds, 2), None);
14159 }
14160
14161 #[test]
14162 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14163 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14165 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14166 }
14167
14168 #[test]
14169 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14170 let rounds = vec![
14171 review_round(false, 1, 2, 2, false, true),
14172 review_round(false, 1, 2, 2, false, true),
14173 ];
14174 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14175 }
14176
14177 fn secs(n: u64) -> Duration {
14178 Duration::from_secs(n)
14179 }
14180
14181 #[tokio::test]
14184 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14185 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14186 if std::env::var_os(CHILD).is_none() {
14187 let output = std::process::Command::new(std::env::current_exe().unwrap())
14191 .args([
14192 "--exact",
14193 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14194 "--nocapture",
14195 ])
14196 .env(CHILD, "1")
14197 .env_remove("MAGI_WORKER_HOME")
14198 .env_remove("MAGI_WORKER_ID")
14199 .output()
14200 .unwrap();
14201 assert!(
14202 output.status.success()
14203 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14204 "{}\n{}",
14205 String::from_utf8_lossy(&output.stdout),
14206 String::from_utf8_lossy(&output.stderr)
14207 );
14208 return;
14209 }
14210 ask_test_home();
14211 let tmp = tempfile::tempdir().unwrap();
14212 let repo = tmp.path().join("repo");
14213 std::fs::create_dir_all(&repo).unwrap();
14214 init_repo(&repo);
14215 let g = |args: &[&str]| {
14216 let out = std::process::Command::new("git")
14217 .args(args)
14218 .current_dir(&repo)
14219 .quiet()
14220 .output()
14221 .unwrap();
14222 assert!(
14223 out.status.success(),
14224 "{}",
14225 String::from_utf8_lossy(&out.stderr)
14226 );
14227 };
14228 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14229 g(&["checkout", "-b", "historical"]);
14230 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14231 g(&["add", "."]);
14232 g(&["commit", "-m", "old alternative"]);
14233 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14234 g(&["checkout", "-b", "magi/task-selected", &base]);
14235 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14236 g(&["add", "."]);
14237 g(&["commit", "-m", "measurement infrastructure"]);
14238 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14239 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14240 g(&["add", "."]);
14241 g(&["commit", "-m", "dependent implementation"]);
14242 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14243 g(&["checkout", "main"]);
14244 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14245 g(&["add", "."]);
14246 g(&["commit", "-m", "unrelated base change"]);
14247 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14248 let origin = tmp.path().join("origin.git");
14249 g(&[
14250 "clone",
14251 "--bare",
14252 repo.to_str().unwrap(),
14253 origin.to_str().unwrap(),
14254 ]);
14255 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14256 g(&["fetch", "origin"]);
14257 let selection = crate::pr_recovery::Selection {
14258 url: "https://example.test/project/pull/1".into(),
14259 branch: "magi/task-selected".into(),
14260 head: head.clone(),
14261 source_head: None,
14262 source_run: None,
14263 source_proof: None,
14264 disposition: Some(crate::pr_recovery::Disposition::Review),
14265 instruction: "preserve measured behavior".into(),
14266 approved_at: jiff::Timestamp::now(),
14267 };
14268 let mut template = runner_at(RunStatus::Reviewing);
14269 template.state.repo = repo.clone();
14270 template.state.base_commit = current.clone();
14271 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14272 template.state.origin = Some(crate::run::Origin::queue("selected"));
14273 template.state.pr_head = Some(selection.branch.clone());
14274 template.state.pr_recovery = Some(selection.clone());
14275 let mut runner =
14276 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14277 .await
14278 .unwrap();
14279 let wt = runner.state.winner().unwrap().worktree.clone();
14280 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14281 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14282 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14283 assert!(runner.state.seeds.is_empty());
14284 assert_eq!(runner.state.instruction, "preserve measured behavior");
14285 runner.sync_to_base().await.unwrap();
14286 assert_eq!(
14287 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14288 "prerequisite"
14289 );
14290 assert_eq!(
14291 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14292 "measured implementation"
14293 );
14294 assert_eq!(
14295 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14296 "current main change"
14297 );
14298 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14299 assert_eq!(
14300 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14301 head
14302 );
14303
14304 let mut partial = runner_at(RunStatus::Reviewing);
14306 partial.state.repo = repo.clone();
14307 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14308 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14309 run: Some("exit 1".into()),
14310 ..Default::default()
14311 }];
14312 let selection = runner.state.pr_recovery.clone().unwrap();
14313 partial.state.pr_recovery = Some(selection.clone());
14314 partial.state.pr_head = Some(selection.branch.clone());
14315 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14316 let id = partial.state.id.clone();
14317 let error = match Runner::open_selected(
14318 &repo,
14319 partial.state,
14320 partial.roles,
14321 1,
14322 runner.state.base_commit.clone(),
14323 selection,
14324 )
14325 .await
14326 {
14327 Ok(_) => panic!("setup must fail"),
14328 Err(e) => e,
14329 };
14330 assert_eq!(
14331 error
14332 .downcast_ref::<crate::run::StartupFailure>()
14333 .unwrap()
14334 .run,
14335 id
14336 );
14337 let failed = RunState::load(&id).unwrap();
14338 assert_eq!(failed.status, RunStatus::Failed);
14339 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14340 assert!(Runner::resume(&id).is_err());
14341 runner.state.status = RunStatus::Ready;
14342 runner.state.driver_exited = true;
14343 runner.state.save().unwrap();
14344 let mut retry = runner_at(RunStatus::Reviewing);
14345 retry.state.repo = repo.clone();
14346 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14347 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14348 reselected.approved_at = jiff::Timestamp::now();
14349 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14350 retry.state.pr_head = Some(reselected.branch.clone());
14351 retry.state.pr_recovery = Some(reselected.clone());
14352 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14353 .await
14354 .unwrap();
14355 let restarted = Runner::open_selected(
14356 &repo,
14357 retry.state,
14358 retry.roles,
14359 1,
14360 runner.state.base_commit.clone(),
14361 reselected,
14362 )
14363 .await
14364 .unwrap();
14365 assert_eq!(
14366 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14367 .await
14368 .unwrap(),
14369 head
14370 );
14371 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14372 assert_eq!(
14373 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14374 head
14375 );
14376 }
14377
14378 #[tokio::test]
14379 #[cfg(unix)]
14380 async fn fallback_and_interrupted_restoration_require_coordination() {
14381 use std::os::fd::AsRawFd as _;
14382 ask_test_home();
14383 let tmp = tempfile::tempdir().unwrap();
14384 let mut runner = registered_runner(tmp.path()).await;
14385 runner.state.config.graph.timeout_resource_wait = 1;
14386 let winner = runner.state.candidates[0].clone();
14387 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14388 let leftover = winner.worktree.join("leftover");
14389 std::fs::write(&leftover, "untrusted edits").unwrap();
14390 let next = tmp.path().join("next-fixer");
14391 let mut failed = spec("first");
14392 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14393 let mut successor = spec("second");
14394 successor.command = vec![
14395 "sh".into(),
14396 "-c".into(),
14397 format!("touch '{}'; echo done", next.display()),
14398 ];
14399 let attempts = vec![
14400 (failed.clone(), "first".into()),
14401 (successor.clone(), "second".into()),
14402 ];
14403 let lock_path = runner.state.repo.join(".git/updater.lock");
14404 let external = std::fs::File::create(&lock_path).unwrap();
14405 assert_eq!(
14406 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14407 0
14408 );
14409 let artifacts = tmp.path().join("artifacts");
14410 let build = |spec, seat| SeatJob {
14411 spec,
14412 seat,
14413 cwd: winner.worktree.clone(),
14414 prompt: "fixture".into(),
14415 timeout: Duration::from_secs(10),
14416 allow_write: true,
14417 sessions: false,
14418 artifacts: artifacts.clone(),
14419 stem: "fix".into(),
14420 handover: None,
14421 };
14422 let (_, _, out) = runner
14423 .ask_fixer_among(
14424 attempts.clone(),
14425 "graft",
14426 None,
14427 Some((&winner.worktree, &head)),
14428 build,
14429 )
14430 .await;
14431 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14432 assert!(leftover.exists(), "contention must prevent clean");
14433 assert!(!next.exists());
14434 assert!(runner.state.handovers.is_empty());
14435 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14437 from_run: "earlier".into(),
14438 generation: 1,
14439 mode: crate::recovery::Mode::VerifyRepair,
14440 reasons: Vec::new(),
14441 reviewed_head: None,
14442 rounds: Vec::new(),
14443 open_findings: Vec::new(),
14444 evidence: None,
14445 criteria_inherited: true,
14446 repair: crate::recovery::Repair::Started {
14447 before: head.clone(),
14448 },
14449 });
14450 assert!(
14451 runner
14452 .recovery_repair(&winner)
14453 .await
14454 .unwrap_err()
14455 .is::<crate::repo_lock::ResourceBlocked>()
14456 );
14457 assert!(matches!(
14458 runner.state.recovery.as_ref().unwrap().repair,
14459 crate::recovery::Repair::Started { .. }
14460 ));
14461 assert!(leftover.exists());
14462 drop(external);
14463 runner.recovery_repair(&winner).await.unwrap();
14464 assert!(!leftover.exists());
14465 assert!(matches!(
14466 runner.state.recovery.as_ref().unwrap().repair,
14467 crate::recovery::Repair::Done { .. }
14468 ));
14469 std::fs::remove_file(&lock_path).unwrap();
14471 std::fs::create_dir(&lock_path).unwrap();
14472 std::fs::write(&leftover, "untrusted edits").unwrap();
14473 let (_, _, out) = runner
14474 .ask_fixer_among(
14475 attempts,
14476 "graft",
14477 None,
14478 Some((&winner.worktree, &head)),
14479 build,
14480 )
14481 .await;
14482 assert!(matches!(out, AgentOutcome::Failed(_)));
14483 assert!(leftover.exists());
14484 assert!(!next.exists());
14485 assert_eq!(
14486 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14487 head
14488 );
14489 }
14490
14491 #[cfg(unix)]
14492 async fn registered_runner(root: &Path) -> Runner {
14493 repository_runner(root, crate::config::test_coordination()).await
14494 }
14495
14496 async fn repository_runner(
14497 root: &Path,
14498 coordination: Option<crate::config::Coordination>,
14499 ) -> Runner {
14500 ask_test_home();
14501 let repo = root.join("repo");
14502 std::fs::create_dir_all(&repo).unwrap();
14503 init_repo(&repo);
14504 if coordination.is_some() {
14505 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14506 }
14507 let mut runner = runner_at(RunStatus::Prep);
14508 runner.state.repo = repo.clone();
14509 runner.state.config.coordination = coordination;
14510 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14511 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14512 runner.state.config.graph.advise = false;
14513 runner.roles.implementers = vec![spec("implement")];
14514 runner.roles.judges = vec![spec("judge")];
14515 runner.prep().await.unwrap();
14516 runner
14517 }
14518
14519 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14520
14521 #[tokio::test]
14522 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14523 let tmp = tempfile::tempdir().unwrap();
14524 let mut runner = repository_runner(tmp.path(), None).await;
14525 let prompts = runner.state.config.prompts.clone();
14526 let run = runner.state.id.clone();
14527 let ctx = WaveCtx {
14528 carry_seats: false,
14529 pauses: None,
14530 run: &run,
14531 node: "advise",
14532 prompts: &prompts,
14533 cache: None,
14534 round: None,
14535 };
14536 let jobs: Vec<_> = ["alpha", "beta"]
14537 .into_iter()
14538 .map(|id| SeatJob {
14539 spec: scripted(id, AUTH_OUTAGE),
14540 seat: SeatState::new(id, id, 7),
14541 prompt: "propose".into(),
14542 cwd: runner.state.candidates[0].worktree.clone(),
14543 timeout: Duration::from_secs(10),
14544 allow_write: false,
14545 sessions: false,
14546 artifacts: agent::artifacts_dir(&runner.state.dir()),
14547 stem: id.into(),
14548 handover: None,
14549 })
14550 .collect();
14551 let mut losses = Vec::new();
14552 let results = ask_wave_with(
14553 jobs.clone(),
14554 runner.sem.clone(),
14555 2,
14556 &[],
14557 &ctx,
14558 &mut losses,
14559 &mut runner.state,
14560 &|_: &str| Ok(()),
14561 )
14562 .await;
14563 assert!(
14564 results
14565 .iter()
14566 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14567 );
14568 assert_eq!(runner.state.quota.len(), 2);
14569 assert!(
14570 runner
14571 .state
14572 .quota
14573 .iter()
14574 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14575 );
14576 assert!(!runner.state.advise_attempted);
14577 runner.stop_for_resource().unwrap();
14578 assert_eq!(runner.state.status, RunStatus::Blocked);
14579 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14580 assert_eq!(
14581 runner.state.resource_block.as_ref().unwrap().kind,
14582 "agent authentication"
14583 );
14584 runner.state.resource_block = None;
14587 runner.state.quota.clear();
14588 let mut mixed = jobs;
14589 mixed[1].spec.command = vec!["false".into()];
14590 let results = ask_wave_with(
14591 mixed,
14592 runner.sem.clone(),
14593 0,
14594 &[],
14595 &ctx,
14596 &mut losses,
14597 &mut runner.state,
14598 &|_: &str| Ok(()),
14599 )
14600 .await;
14601 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14602 assert!(runner.state.resource_block.is_none());
14603 assert_eq!(losses.len(), 1);
14604 }
14605
14606 #[tokio::test]
14607 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14608 let tmp = tempfile::tempdir().unwrap();
14609 let mut runner = repository_runner(tmp.path(), None).await;
14610 let implementer = scripted("implement", AUTH_OUTAGE);
14611 runner.roles.implementers = vec![implementer.clone()];
14612 runner.state.config.agents = vec![implementer];
14613 runner.implement().await.unwrap();
14614 assert_eq!(runner.state.status, RunStatus::Blocked);
14615 assert_eq!(runner.state.quota.len(), 1);
14616 assert_eq!(
14617 runner.state.quota[0].cause,
14618 agent::QuotaCause::Authentication
14619 );
14620 let candidate = &runner.state.candidates[0];
14621 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14622 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14623 }
14624
14625 #[tokio::test]
14626 #[cfg(unix)]
14627 async fn parked_implementer_persists_completed_work() {
14628 for registered in [false, true] {
14629 let tmp = tempfile::tempdir().unwrap();
14630 let mut runner = registered_runner(tmp.path()).await;
14631 if !registered {
14632 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14633 }
14634 let marker = tmp.path().join("started");
14635 let release = tmp.path().join("release");
14636 let mut implementer = spec("implement");
14637 implementer.command = vec![
14638 "sh".into(),
14639 "-c".into(),
14640 format!(
14641 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14642 marker.display(),
14643 release.display()
14644 ),
14645 ];
14646 runner.roles.implementers = vec![implementer.clone()];
14647 runner.state.config.agents = vec![implementer];
14648 let pause = runner.pause.clone();
14649 let (result, ()) = tokio::join!(runner.implement(), async {
14650 tokio::time::timeout(Duration::from_secs(10), async {
14651 while !marker.exists() {
14652 tokio::time::sleep(Duration::from_millis(10)).await;
14653 }
14654 })
14655 .await
14656 .unwrap();
14657 pause.park();
14658 std::fs::write(&release, "go").unwrap();
14659 });
14660 result.unwrap();
14661 let candidate = &runner.state.candidates[0];
14662 assert!(candidate.commits > 0, "{candidate:?}");
14663 assert!(candidate.summary.contains("saved-result"));
14664 assert_eq!(candidate.files, 1);
14665 let before = candidate.commits;
14666 runner.implement().await.unwrap();
14667 assert_eq!(runner.state.candidates[0].commits, before);
14668 }
14669 }
14670
14671 #[tokio::test]
14672 #[cfg(unix)]
14673 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14674 let tmp = tempfile::tempdir().unwrap();
14675 let mut runner = registered_runner(tmp.path()).await;
14676 let winner = runner.state.candidates[0].clone();
14677 let cache = tmp.path().join("cache");
14678 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14679 runner.state.config.graph.timeout_resource_wait = 0;
14680 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14681 let crate::cache::AcquireOutcome::Acquired(held) =
14682 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14683 else {
14684 panic!("cache must be free")
14685 };
14686 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14687 from_run: "earlier".into(),
14688 generation: 1,
14689 mode: crate::recovery::Mode::VerifyRepair,
14690 reasons: Vec::new(),
14691 reviewed_head: None,
14692 rounds: Vec::new(),
14693 open_findings: Vec::new(),
14694 evidence: None,
14695 criteria_inherited: true,
14696 repair: crate::recovery::Repair::NotStarted,
14697 });
14698 runner.recovery_repair(&winner).await.unwrap();
14699 assert!(matches!(
14700 runner.state.recovery.as_ref().unwrap().repair,
14701 crate::recovery::Repair::WaitingForRestore { .. }
14702 ));
14703 drop(held);
14704 runner.state.resource_block = None;
14705 runner.state.config.graph.timeout_resource_wait = 1;
14706 runner.recovery_repair(&winner).await.unwrap();
14707 assert!(matches!(
14708 runner.state.recovery.as_ref().unwrap().repair,
14709 crate::recovery::Repair::Done { .. }
14710 ));
14711 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14712 }
14713
14714 #[tokio::test]
14715 #[cfg(unix)]
14716 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14717 use std::os::fd::AsRawFd as _;
14718 for graft in [false, true] {
14719 let tmp = tempfile::tempdir().unwrap();
14720 let mut runner = registered_runner(tmp.path()).await;
14721 runner.state.config.graph.timeout_resource_wait = 1;
14722 let winner = runner.state.candidates[0].clone();
14723 let marker = tmp.path().join("fixer-started");
14724 let mut fixer = spec("fixer");
14725 fixer.command = vec![
14726 "sh".into(),
14727 "-c".into(),
14728 format!("echo started >> '{}'; echo done", marker.display()),
14729 ];
14730 runner.roles.fixer = Some(vec![fixer]);
14731 if graft {
14732 let mut loser = winner.clone();
14733 loser.label = 'B';
14734 loser.index = 1;
14735 runner.state.candidates.push(loser);
14736 runner.state.config.graph.graft_runner_up = true;
14737 runner.state.tally = Some(Tally {
14738 first_choice: BTreeMap::new(),
14739 borda: BTreeMap::new(),
14740 winner: winner.label,
14741 rankings: 1,
14742 unanimous_initial: true,
14743 deliberated: false,
14744 changed_votes: 0,
14745 unanimous_final: true,
14746 tie_break: None,
14747 judges: 1,
14748 present: 1,
14749 quorum: 1,
14750 met_quorum: true,
14751 uncontested: None,
14752 });
14753 runner.state.judgements.push(crate::run::Judgement {
14754 judge: 1,
14755 seat: "judge-1".into(),
14756 agent: "judge".into(),
14757 ranking: vec![winner.label, 'B'],
14758 reasons: BTreeMap::new(),
14759 confidence: None,
14760 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14761 order: vec![0, 1],
14762 failed: None,
14763 duration_ms: 0,
14764 });
14765 } else {
14766 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14767 from_run: "earlier".into(),
14768 generation: 1,
14769 mode: crate::recovery::Mode::VerifyRepair,
14770 reasons: Vec::new(),
14771 reviewed_head: None,
14772 rounds: Vec::new(),
14773 open_findings: Vec::new(),
14774 evidence: None,
14775 criteria_inherited: true,
14776 repair: crate::recovery::Repair::NotStarted,
14777 });
14778 }
14779 let reader = crate::repo_lock::SeatLease::for_checkout(
14780 &winner.worktree,
14781 false,
14782 crate::config::test_coordination().as_ref(),
14783 )
14784 .await
14785 .unwrap();
14786 let pause = runner.pause.clone();
14787 let (result, ()) = tokio::join!(
14788 async {
14789 if graft {
14790 runner.graft().await
14791 } else {
14792 runner.recovery_repair(&winner).await
14793 }
14794 },
14795 async {
14796 tokio::time::sleep(Duration::from_millis(100)).await;
14797 pause.park();
14798 }
14799 );
14800 result.unwrap();
14801 assert!(!marker.exists(), "a blocked seat must not spawn");
14802 runner.state = RunState::load(&runner.state.id).unwrap();
14804 if graft {
14805 assert_eq!(
14806 runner.state.graft.as_ref().unwrap().outcome,
14807 GraftOutcome::WaitingForRestore
14808 );
14809 } else {
14810 assert!(matches!(
14811 runner.state.recovery.as_ref().unwrap().repair,
14812 crate::recovery::Repair::WaitingForRestore { .. }
14813 ));
14814 }
14815 drop(reader);
14816 runner.pause = Pause::new();
14817 runner.state.resource_block = None;
14818 let external =
14820 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14821 assert_eq!(
14822 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14823 0
14824 );
14825 let leftover = winner.worktree.join("leftover");
14826 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14827 let result = if graft {
14828 runner.graft().await
14829 } else {
14830 runner.recovery_repair(&winner).await
14831 };
14832 assert!(
14833 result
14834 .unwrap_err()
14835 .is::<crate::repo_lock::ResourceBlocked>()
14836 );
14837 assert!(leftover.exists());
14838 assert!(!marker.exists());
14839 drop(external);
14840 runner.state.resource_block = None;
14841 let result = if graft {
14842 runner.graft().await
14843 } else {
14844 runner.recovery_repair(&winner).await
14845 };
14846 result.unwrap();
14847 assert!(!leftover.exists(), "restore must precede retry");
14848 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14849 if graft {
14850 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14851 } else {
14852 assert!(matches!(
14853 runner.state.recovery.as_ref().unwrap().repair,
14854 crate::recovery::Repair::Done { .. }
14855 ));
14856 }
14857 }
14858 }
14859
14860 #[tokio::test]
14861 #[cfg(unix)]
14862 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14863 use std::os::fd::AsRawFd as _;
14864 let tmp = tempfile::tempdir().unwrap();
14865 let mut runner = registered_runner(tmp.path()).await;
14866 runner.state.config.graph.timeout_resource_wait = 1;
14867 let winner = runner.state.candidates[0].clone();
14868 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14869 runner.state.tally = Some(Tally {
14870 first_choice: BTreeMap::new(),
14871 borda: BTreeMap::new(),
14872 winner: winner.label,
14873 rankings: 0,
14874 unanimous_initial: false,
14875 deliberated: false,
14876 changed_votes: 0,
14877 unanimous_final: false,
14878 tie_break: None,
14879 judges: 0,
14880 present: 0,
14881 quorum: 0,
14882 met_quorum: true,
14883 uncontested: Some("fixture".into()),
14884 });
14885 runner.state.graft = Some(GraftRecord {
14886 pre: pre.clone(),
14887 post: None,
14888 lines: None,
14889 limit: None,
14890 agent: None,
14891 outcome: GraftOutcome::Running,
14892 requested: Vec::new(),
14893 adoption: None,
14894 adoption_error: None,
14895 });
14896 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14897 git::commit_all(&winner.worktree, "unfinished graft")
14898 .await
14899 .unwrap();
14900 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14901 let leftover = winner.worktree.join("leftover");
14902 std::fs::write(&leftover, "untrusted edits").unwrap();
14903 let path = runner.state.repo.join(".git/updater.lock");
14904 let external = std::fs::File::create(&path).unwrap();
14905 assert_eq!(
14906 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14907 0
14908 );
14909 assert!(
14910 runner
14911 .graft()
14912 .await
14913 .unwrap_err()
14914 .is::<crate::repo_lock::ResourceBlocked>()
14915 );
14916 assert!(matches!(
14917 runner.state.graft.as_ref().unwrap().outcome,
14918 GraftOutcome::Running
14919 ));
14920 assert_eq!(
14921 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14922 unfinished
14923 );
14924 assert!(leftover.exists());
14925 drop(external);
14926 std::fs::remove_file(&path).unwrap();
14927 std::fs::create_dir(&path).unwrap();
14928 assert!(runner.graft().await.is_err());
14929 assert!(matches!(
14930 runner.state.graft.as_ref().unwrap().outcome,
14931 GraftOutcome::Running
14932 ));
14933 assert_eq!(
14934 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14935 unfinished
14936 );
14937 assert!(leftover.exists());
14938 std::fs::remove_dir(&path).unwrap();
14939 runner.graft().await.unwrap();
14940 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14941 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14942 assert!(!leftover.exists());
14943 }
14944
14945 #[tokio::test]
14946 #[cfg(unix)]
14947 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14948 use std::os::fd::AsRawFd as _;
14949 let tmp = tempfile::tempdir().unwrap();
14950 let mut runner = registered_runner(tmp.path()).await;
14951 runner.state.prep_complete = Some(false);
14952 runner.state.config.graph.timeout_resource_wait = 1;
14953 runner.state.save().unwrap();
14954 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14955 assert_eq!(
14956 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14957 0
14958 );
14959 runner.execute().await.unwrap();
14960 let disk = RunState::load(&runner.state.id).unwrap();
14961 assert!(disk.driver_exited);
14962 assert_eq!(disk.driver_pid, None);
14963 assert_eq!(runner.state.status, RunStatus::Blocked);
14964 assert_eq!(
14965 runner.state.resource_block.as_ref().unwrap().kind,
14966 "repository coordination"
14967 );
14968 assert!(runner.state.prep_failure.is_none());
14969 assert!(runner.state.seats.is_empty());
14970 runner.pause.park();
14971 tokio::time::timeout(Duration::from_secs(10), runner.execute())
14974 .await
14975 .unwrap()
14976 .unwrap();
14977 assert!(runner.state.parked);
14978 assert!(runner.state.prep_failure.is_none());
14979 let disk = RunState::load(&runner.state.id).unwrap();
14980 assert!(disk.driver_exited);
14981 assert_eq!(disk.driver_pid, None);
14982 }
14983
14984 #[tokio::test]
14985 #[cfg(unix)]
14986 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14987 use std::os::fd::AsRawFd as _;
14988 let tmp = tempfile::tempdir().unwrap();
14989 let mut runner = registered_runner(tmp.path()).await;
14990 runner.state.prep_complete = Some(false);
14991 runner.state.config.graph.timeout_resource_wait = 30;
14992 runner.state.save().unwrap();
14993 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14994 assert_eq!(
14995 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14996 0
14997 );
14998 assert!(
14999 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15000 .await
15001 .is_err()
15002 );
15003 let disk = RunState::load(&runner.state.id).unwrap();
15004 assert!(disk.driver_exited);
15005 assert_eq!(disk.driver_pid, None);
15006 assert!(disk.status.resumable());
15007 assert!(runner.state.driver_exited);
15008 assert_eq!(runner.state.driver_pid, None);
15009 runner.state.save().unwrap();
15010 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15011 }
15012
15013 #[tokio::test]
15014 #[cfg(unix)]
15015 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15016 let tmp = tempfile::tempdir().unwrap();
15017 let mut runner = registered_runner(tmp.path()).await;
15018 let winner = runner.state.candidates[0].clone();
15019 let cache = tmp.path().join("cache");
15020 let started = tmp.path().join("pre-gate-started");
15021 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15022 runner.state.config.verify.gate =
15023 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15024 let home = crate::run::home();
15025 let owner =
15026 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15027 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15028 crate::cache::AcquireOutcome::Acquired(g) => g,
15029 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15030 };
15031 let mut gate = Box::pin(runner.run_gate(&winner));
15032 assert!(
15033 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15034 .await
15035 .is_err()
15036 );
15037 assert!(!started.exists(), "pre-gate must wait for the cache too");
15038 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15042 let editor = crate::repo_lock::SeatLease::checkout_with(
15043 &winner.worktree,
15044 true,
15045 crate::config::test_coordination().as_ref(),
15046 &wait,
15047 )
15048 .await
15049 .unwrap();
15050 drop(editor);
15051 guard.release();
15052 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15053 .await
15054 .unwrap()
15055 .unwrap();
15056 drop(gate);
15057 assert!(outcomes.iter().all(CommandOutcome::ok));
15058 assert!(started.exists());
15059 }
15060
15061 #[tokio::test]
15062 #[cfg(unix)]
15063 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15064 use std::os::fd::AsRawFd as _;
15065
15066 ask_test_home();
15067 let tmp = tempfile::tempdir().unwrap();
15068 let repo = tmp.path().join("repo");
15069 std::fs::create_dir_all(&repo).unwrap();
15070 init_repo(&repo);
15071 let common = repo.join(".git");
15072 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15073 let mut runner = runner_at(RunStatus::Prep);
15074 runner.state.repo = repo.clone();
15075 runner.state.config.coordination = crate::config::test_coordination();
15076 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15077 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15078 runner.state.config.graph.advise = false;
15079 runner.roles.implementers = vec![spec("implement")];
15080 runner.roles.judges = vec![spec("judge")];
15081 runner.prep().await.unwrap();
15082 let winner = runner.state.candidates[0].clone();
15083 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15086 let completed = tmp.path().join("seat-completed");
15087 runner.roles.implementers[0].command = vec![
15088 "sh".into(),
15089 "-c".into(),
15090 format!("touch '{}'; echo completed", completed.display()),
15091 ];
15092 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15093 assert_eq!(
15094 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15095 0
15096 );
15097 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15098 let mut implement = Box::pin(runner.implement());
15099 tokio::select! {
15100 result = &mut implement => panic!("rescue must wait: {result:?}"),
15101 result = tokio::time::timeout(Duration::from_secs(10), async {
15102 while !completed.exists() {
15103 tokio::time::sleep(Duration::from_millis(20)).await;
15104 }
15105 }) => result.unwrap(),
15106 }
15107 assert!(
15108 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15109 .await
15110 .is_err()
15111 );
15112 assert_eq!(
15113 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15114 before
15115 );
15116 assert_eq!(
15117 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15118 0
15119 );
15120 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15121 .await
15122 .unwrap()
15123 .unwrap();
15124 drop(implement);
15125 assert_ne!(
15126 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15127 before
15128 );
15129 assert!(git::is_clean(&winner.worktree).await.unwrap());
15130
15131 let started = tmp.path().join("pre-gate-started");
15132 let proceed = tmp.path().join("pre-gate-proceed");
15133 let checked = tmp.path().join("gate-ran");
15134 runner.state.config.verify.pre_gate = vec![format!(
15135 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15136 started.display(),
15137 proceed.display()
15138 )];
15139 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15140 assert_eq!(
15141 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15142 0
15143 );
15144 let mut gate = Box::pin(runner.run_gate(&winner));
15145 assert!(
15146 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15147 .await
15148 .is_err()
15149 );
15150 assert!(!started.exists());
15151 assert!(!checked.exists());
15152 assert!(git::is_clean(&winner.worktree).await.unwrap());
15153 assert_eq!(
15154 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15155 0
15156 );
15157 tokio::select! {
15158 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15159 result = tokio::time::timeout(Duration::from_secs(10), async {
15160 while !started.exists() {
15161 tokio::time::sleep(Duration::from_millis(20)).await;
15162 }
15163 }) => result.unwrap(),
15164 }
15165 assert_eq!(
15166 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15167 -1,
15168 "mechanical edits must hold the real lock"
15169 );
15170 std::fs::write(proceed, "ready").unwrap();
15171 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15172 .await
15173 .unwrap()
15174 .unwrap();
15175 drop(gate);
15176 assert!(outcomes.iter().all(CommandOutcome::ok));
15177 assert!(checked.exists());
15178 assert!(runner.state.pre_gate_commit.is_some());
15179 assert!(git::is_clean(&winner.worktree).await.unwrap());
15180 assert_eq!(
15181 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15182 0,
15183 "gate completion must release coordination"
15184 );
15185 drop(outsider);
15186 std::fs::remove_file(common.join("updater.lock")).unwrap();
15187 std::fs::create_dir(common.join("updater.lock")).unwrap();
15188 std::fs::remove_file(&started).unwrap();
15189 std::fs::remove_file(&checked).unwrap();
15190 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15191 let error = runner.run_gate(&winner).await.unwrap_err();
15192 assert!(
15193 error
15194 .to_string()
15195 .contains("acquire mandatory repository lock")
15196 );
15197 assert!(
15198 !started.exists(),
15199 "failed acquisition must prevent pre-gate"
15200 );
15201 assert!(!checked.exists(), "failed acquisition must prevent gate");
15202 assert!(git::is_clean(&winner.worktree).await.unwrap());
15203 assert_eq!(
15204 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15205 head
15206 );
15207 }
15208
15209 #[tokio::test]
15210 #[cfg(unix)]
15211 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15212 use std::os::fd::AsRawFd as _;
15213 use std::time::Duration;
15214
15215 ask_test_home();
15216 let tmp = tempfile::tempdir().unwrap();
15217 let repo = tmp.path().join("repo");
15218 std::fs::create_dir_all(&repo).unwrap();
15219 init_repo(&repo);
15220 let common = repo.join(".git");
15221 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15222 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15223 assert_eq!(
15224 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15225 0
15226 );
15227 let mut runner = runner_at(RunStatus::Prep);
15228 runner.state.repo = repo.clone();
15229 runner.state.config.coordination = crate::config::test_coordination();
15230 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15231 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15232 runner.state.config.graph.advise = true;
15233 runner.state.config.graph.advisors = 1;
15234 runner.roles.implementers = vec![spec("implement")];
15235 runner.roles.judges = vec![spec("judge")];
15236 let root = runner.state.worktree_root();
15237 let started = tmp.path().join("setup-started");
15238 let proceed = tmp.path().join("setup-proceed");
15239 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15240 run: Some(format!(
15241 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15242 started.display(),
15243 proceed.display()
15244 )),
15245 ..Default::default()
15246 }];
15247 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15248 let config = git::git(&repo, &["config", "--local", "--list"])
15249 .await
15250 .unwrap();
15251 let mut prep = Box::pin(runner.prep());
15252 assert!(
15253 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15254 .await
15255 .is_err()
15256 );
15257 assert!(
15258 !root.exists(),
15259 "contention must prevent even checkout creation"
15260 );
15261 assert!(!started.exists(), "contention must prevent setup");
15262 assert_eq!(
15263 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15264 branches
15265 );
15266 assert_eq!(
15267 git::git(&repo, &["config", "--local", "--list"])
15268 .await
15269 .unwrap(),
15270 config
15271 );
15272 drop(lock);
15273 tokio::select! {
15274 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15275 result = tokio::time::timeout(Duration::from_secs(10), async {
15276 while !started.exists() {
15277 tokio::time::sleep(Duration::from_millis(20)).await;
15278 }
15279 }) => result.unwrap(),
15280 }
15281 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15282 assert_eq!(
15283 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15284 -1
15285 );
15286 assert!(
15287 tokio::time::timeout(
15288 Duration::from_millis(100),
15289 crate::repo_lock::SeatLease::for_checkout(
15290 &repo,
15291 true,
15292 crate::config::test_coordination().as_ref()
15293 )
15294 )
15295 .await
15296 .is_err(),
15297 "preparation must also exclude local editors"
15298 );
15299 std::fs::write(proceed, "ready").unwrap();
15300 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15301 .await
15302 .unwrap()
15303 .unwrap();
15304 drop(prep);
15305 assert_eq!(runner.state.prep_complete, Some(true));
15306 assert!(root.join("judge-1").exists());
15307 assert!(root.join("advisor-1").exists());
15308 assert_eq!(
15309 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15310 0
15311 );
15312 }
15313
15314 #[tokio::test]
15315 #[cfg(unix)]
15316 async fn preparation_lock_failure_prevents_repository_changes() {
15317 ask_test_home();
15318 let tmp = tempfile::tempdir().unwrap();
15319 let repo = tmp.path().join("repo");
15320 std::fs::create_dir_all(&repo).unwrap();
15321 init_repo(&repo);
15322 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15323 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15326 let mut runner = runner_at(RunStatus::Prep);
15327 runner.state.repo = repo.clone();
15328 runner.state.config.coordination = crate::config::test_coordination();
15329 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15330 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15331 runner.roles.implementers = vec![spec("implement")];
15332 let error = runner.prep().await.unwrap_err();
15333 assert!(
15334 error
15335 .to_string()
15336 .contains("acquire mandatory repository lock")
15337 );
15338 assert!(!runner.state.worktree_root().exists());
15339 assert!(runner.state.candidates.is_empty());
15340 assert!(!repo.join(".git/config.worktree").exists());
15341 assert_eq!(
15342 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15343 "* main"
15344 );
15345 }
15346
15347 #[tokio::test]
15348 async fn preparation_records_creation_intent_before_git_can_fail() {
15349 ask_test_home();
15350 let tmp = tempfile::tempdir().unwrap();
15351 let repo = tmp.path().join("repo");
15352 std::fs::create_dir_all(&repo).unwrap();
15353 init_repo(&repo);
15354 let mut runner = runner_at(RunStatus::Prep);
15355 runner.state.repo = repo.clone();
15356 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15357 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15358 runner.state.config.blind.commit_msg_hook = false;
15359 runner.roles.implementers = vec![spec("one")];
15360 let label = blind::assign_labels(1, runner.state.seed)[0];
15361 let branch = runner.state.branch_for(label);
15362 git::git(&repo, &["branch", &branch]).await.unwrap();
15365 assert!(runner.prep().await.is_err());
15366 let saved = RunState::load(&runner.state.id).unwrap();
15367 assert!(saved.candidates.is_empty());
15368 let event = saved
15369 .events
15370 .iter()
15371 .find(|e| e.node == "prep-intent")
15372 .unwrap();
15373 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15374 assert_eq!(intent["branch"], branch);
15375 assert_eq!(intent["start"], runner.state.base_commit);
15376 assert_eq!(
15377 intent["worktree"],
15378 runner
15379 .state
15380 .worktree_root()
15381 .join(format!("cand-{label}"))
15382 .to_string_lossy()
15383 .as_ref()
15384 );
15385 assert_eq!(intent["picks"], serde_json::json!([]));
15386 }
15387
15388 #[tokio::test]
15389 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15390 ask_test_home();
15391 let tmp = tempfile::tempdir().unwrap();
15392 let repo = tmp.path().join("repo");
15393 std::fs::create_dir_all(&repo).unwrap();
15394 init_repo(&repo);
15395 let mut runner = runner_at(RunStatus::Prep);
15396 runner.state.repo = repo.clone();
15397 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15398 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15399 runner.state.sources.pick_commits.push(refs::Input {
15400 token: "missing".into(),
15401 sha: "0".repeat(40),
15402 provenance: refs::Provenance::ExplicitCli,
15403 });
15404 assert!(runner.prep().await.is_err());
15405 assert!(runner.state.candidates.is_empty());
15406 assert!(!runner.state.worktree_root().exists());
15407 }
15408
15409 #[tokio::test]
15410 async fn prose_and_legacy_inference_never_import_into_candidates() {
15411 ask_test_home();
15412 let tmp = tempfile::tempdir().unwrap();
15413 let repo = tmp.path().join("repo");
15414 std::fs::create_dir_all(&repo).unwrap();
15415 init_repo(&repo);
15416 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15417 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15418 .await
15419 .unwrap();
15420 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15421 git::git(&repo, &["add", "."]).await.unwrap();
15422 git::git(&repo, &["commit", "-m", "historical implementation"])
15423 .await
15424 .unwrap();
15425 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15426 git::git(&repo, &["checkout", "main"]).await.unwrap();
15427 for instruction in [
15428 format!("Example: magi/history/A at {source}"),
15429 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15430 format!("Never import magi/history/A or {source}"),
15431 format!("Conversation answer: I mentioned {source} only as background"),
15432 ] {
15433 let mut runner = runner_at(RunStatus::Prep);
15434 runner.state.repo = repo.clone();
15435 runner.state.base_commit = base.clone();
15436 runner.state.instruction = instruction.clone();
15437 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15438 runner.state.config.blind.commit_msg_hook = false;
15439 runner.state.seeds = vec![refs::Seed {
15440 token: "magi/history/A".into(),
15441 sha: source.clone(),
15442 kind: refs::SeedKind::Unmerged,
15443 branch: true,
15444 detail: String::new(),
15445 }];
15446 runner.roles.implementers = vec![spec("one")];
15447 assert_eq!(seeded_instruction(&runner.state), instruction);
15448 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15450 legacy.as_object_mut().unwrap().remove("sources");
15451 runner.state = serde_json::from_value(legacy).unwrap();
15452 assert_eq!(runner.state.sources, refs::Sources::default());
15453 runner.state.save().unwrap();
15454 runner.state = RunState::load(&runner.state.id).unwrap();
15455 runner.prep().await.unwrap();
15456 for candidate in &runner.state.candidates {
15457 assert_eq!(
15458 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15459 base
15460 );
15461 assert!(!candidate.worktree.join("unwanted").exists());
15462 }
15463 }
15464 }
15465
15466 #[tokio::test]
15467 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15468 ask_test_home();
15469 let tmp = tempfile::tempdir().unwrap();
15470 let repo = tmp.path().join("repo");
15471 std::fs::create_dir_all(&repo).unwrap();
15472 init_repo(&repo);
15473 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15474 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15475 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15476 git::git(&repo, &["add", "."]).await.unwrap();
15477 git::git(&repo, &["commit", "-m", "required seed"])
15478 .await
15479 .unwrap();
15480 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15481 git::git(&repo, &["checkout", "main"]).await.unwrap();
15482 let mut runner = runner_at(RunStatus::Prep);
15483 runner.state.repo = repo.clone();
15484 runner.state.base_commit = base.clone();
15485 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15486 runner.state.config.graph.advise = false;
15487 runner.state.config.blind.commit_msg_hook = false;
15488 runner.state.sources =
15489 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15490 .await
15491 .unwrap();
15492 runner.roles.implementers = vec![spec("one"), spec("two")];
15493 runner.roles.judges = vec![spec("judge")];
15494 let label = blind::assign_labels(2, runner.state.seed)[0];
15495 let branch = runner.state.branch_for(label);
15496 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15497 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15498 .await
15499 .unwrap();
15500 runner.state.candidates.push(Candidate {
15501 index: 0,
15502 label,
15503 agent: "one".into(),
15504 branch,
15505 worktree,
15506 summary: String::new(),
15507 stat: String::new(),
15508 files: 0,
15509 commits: 0,
15510 empty: false,
15511 failed: None,
15512 verified_noop: None,
15513 duration_ms: 0,
15514 folded: false,
15515 });
15516 runner.state.save().unwrap();
15517 runner.state = RunState::load(&runner.state.id).unwrap();
15519 assert_eq!(runner.state.prep_complete, Some(false));
15520 runner.prep().await.unwrap();
15521 assert_eq!(runner.state.prep_complete, Some(true));
15522 assert_eq!(runner.state.candidates.len(), 2);
15523 for candidate in &runner.state.candidates {
15524 assert_eq!(
15525 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15526 git::tree_of(&repo, &seed).await.unwrap()
15527 );
15528 assert_eq!(
15529 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15530 "required commit"
15531 );
15532 }
15533 assert!(runner.state.worktree_root().join("judge-1").exists());
15534 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15535 runner.prep().await.unwrap();
15536 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15537
15538 runner.state.prep_complete = Some(false);
15539 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15540 run: Some(format!(
15541 "test -f '{}'",
15542 tmp.path().join("dependency-ready").display()
15543 )),
15544 ..Default::default()
15545 }];
15546 assert!(runner.prep().await.is_err());
15547 assert!(runner.state.prep_failure.is_some());
15548 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15549 runner.state = RunState::load(&runner.state.id).unwrap();
15550 assert!(
15551 runner.prep().await.is_err(),
15552 "correcting the cause alone must not auto-retry"
15553 );
15554 let failed_at = runner.state.events.last().unwrap().at;
15555 assert!(
15556 runner.retry_preparation(failed_at).is_err(),
15557 "old approvals cannot unlock the hold"
15558 );
15559 let mut task = crate::queue::Task::new(
15560 "retry".into(),
15561 "retry setup".into(),
15562 repo.clone(),
15563 crate::queue::Source::Human,
15564 );
15565 task.start(runner.state.id.clone());
15566 task.hold_machine(Some("setup failed".into()));
15567 task.release_operator("cli", None);
15568 runner
15569 .retry_preparation(task.resumption.as_ref().unwrap().at)
15570 .unwrap();
15571 runner.prep().await.unwrap();
15572 assert!(runner.state.prep_failure.is_none());
15573 assert_eq!(runner.state.prep_complete, Some(true));
15574 assert_eq!(runner.state.candidates.len(), 2);
15575 assert!(runner.state.worktree_root().join("judge-1").exists());
15576 for candidate in &runner.state.candidates {
15577 assert_eq!(
15578 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15579 git::tree_of(&repo, &seed).await.unwrap()
15580 );
15581 }
15582 }
15583
15584 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15585 let mut heads = Vec::new();
15586 for candidate in candidates {
15587 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15588 }
15589 heads
15590 }
15591
15592 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15593 ask_test_home();
15594 let tmp = tempfile::tempdir().unwrap();
15595 let repo = tmp.path().join("repo");
15596 std::fs::create_dir(&repo).unwrap();
15597 init_repo(&repo);
15598 let mut runner = runner_at(RunStatus::Reviewing);
15599 runner.state.repo = repo.clone();
15600 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15601 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15602 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15603 let branch = format!("candidate-{label}");
15604 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15605 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15606 .await
15607 .unwrap();
15608 if label == 'B' {
15609 std::fs::write(worktree.join("change"), "winner").unwrap();
15610 git::rescue_commit(&worktree, "candidate change")
15611 .await
15612 .unwrap();
15613 }
15614 runner.state.candidates.push(Candidate {
15615 index,
15616 label,
15617 agent: "alpha".into(),
15618 branch,
15619 worktree,
15620 summary: "completed reply".into(),
15621 stat: String::new(),
15622 files: usize::from(label == 'B'),
15623 commits: usize::from(label == 'B'),
15624 empty: label == 'A',
15625 failed: None,
15626 verified_noop: None,
15627 duration_ms: 1,
15628 folded: false,
15629 });
15630 }
15631 runner.state.judge_skipped = true;
15632 runner.state.tally = Some(Tally {
15633 first_choice: BTreeMap::new(),
15634 borda: BTreeMap::new(),
15635 winner: 'B',
15636 rankings: 0,
15637 unanimous_initial: false,
15638 deliberated: false,
15639 changed_votes: 0,
15640 unanimous_final: false,
15641 tie_break: None,
15642 judges: 0,
15643 present: 0,
15644 quorum: 0,
15645 met_quorum: true,
15646 uncontested: Some("only B produced a change".into()),
15647 });
15648 runner.state.save().unwrap();
15649 (tmp, runner)
15650 }
15651
15652 #[tokio::test]
15653 #[cfg(unix)]
15654 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15655 for (solo, failed_checkpoint) in
15656 [(true, false), (false, false), (true, true), (false, true)]
15657 {
15658 let (_tmp, mut runner) = preservation_resume_fixture().await;
15659 if solo {
15660 runner.state.candidates.retain(|c| c.label == 'A');
15661 }
15662 runner.state.candidates[0].empty = false;
15663 runner.state.candidates[0].summary.clear();
15664 runner.state.tally = None;
15665 runner.state.judge_skipped = false;
15666 runner.state.status = RunStatus::Implementing;
15667 runner.state.config.disk.min_free_bytes = u64::MAX;
15668 let mut implementer = spec("alpha");
15669 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15670 runner.roles.implementers = vec![implementer];
15671 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15672 if failed_checkpoint {
15673 std::fs::create_dir_all(&patch).unwrap();
15674 }
15675 runner.state.save().unwrap();
15676 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15677 runner.state = RunState::load(&runner.state.id).unwrap();
15678 assert_eq!(runner.state.status, RunStatus::Blocked);
15679 assert!(runner.state.interrupted_implementations.contains(&'A'));
15680 assert_eq!(runner.state.candidates[0].commits, 0);
15681 assert!(!runner.state.candidates[0].empty);
15682 assert!(runner.state.candidates[0].failed.is_none());
15683 if failed_checkpoint {
15684 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15685 std::fs::remove_dir(patch).unwrap();
15686 }
15687 runner.state.resource_block = None;
15688 runner.state.config.disk.min_free_bytes = 0;
15689 runner.roles.implementers[0].command = vec![
15690 "sh".into(),
15691 "-c".into(),
15692 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15693 ];
15694 runner.implement().await.unwrap();
15695 assert!(runner.state.interrupted_implementations.is_empty());
15696 assert!(runner.state.candidate_preservation.is_empty());
15697 assert_eq!(runner.state.candidates[0].commits, 1);
15698 assert!(!runner.state.candidates[0].empty);
15699 assert!(
15700 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15701 .unwrap()
15702 .contains("retried empty interrupted seat")
15703 );
15704 if !solo {
15705 assert_eq!(runner.state.candidates[1].commits, 1);
15706 }
15707 }
15708 }
15709
15710 #[tokio::test]
15711 #[cfg(unix)]
15712 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15713 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15714 let (_tmp, mut runner) = preservation_resume_fixture().await;
15715 runner.state.candidates.retain(|c| c.label == 'A');
15716 runner.state.candidates[0].empty = false;
15717 runner.state.candidates[0].summary.clear();
15718 runner.state.tally = None;
15719 runner.state.judge_skipped = false;
15720 runner.state.status = RunStatus::Implementing;
15721 runner.state.config.disk.min_free_bytes = u64::MAX;
15724 let wt = runner.state.candidates[0].worktree.clone();
15725 std::fs::write(
15726 wt.join("interrupted-source"),
15727 "retain implementation work\n",
15728 )
15729 .unwrap();
15730 if committed {
15731 git::rescue_commit(&wt, "implementation checkpoint")
15732 .await
15733 .unwrap();
15734 }
15735 let lock = git::git(
15736 &wt,
15737 &[
15738 "rev-parse",
15739 "--path-format=absolute",
15740 "--git-path",
15741 "index.lock",
15742 ],
15743 )
15744 .await
15745 .unwrap();
15746 if block_rescue {
15747 std::fs::write(&lock, "fixture-owned lock").unwrap();
15748 }
15749 let mut implementer = spec("alpha");
15750 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15751 runner.roles.implementers = vec![implementer];
15752 runner.state.save().unwrap();
15753 let result = runner.implement().await;
15754 assert_eq!(result.is_err(), block_rescue);
15755 assert_eq!(
15756 runner.state.resource_block.as_ref().unwrap().kind,
15757 "disk pressure"
15758 );
15759 assert!(
15760 runner
15761 .state
15762 .events
15763 .iter()
15764 .any(|e| e.message.contains("agent execution interrupted"))
15765 );
15766 assert!(
15767 runner
15768 .state
15769 .dir()
15770 .join("artifacts/candidate-A-recovery/staged.patch")
15771 .exists()
15772 );
15773 if block_rescue {
15774 let saved = RunState::load(&runner.state.id).unwrap();
15775 assert!(saved.candidate_preservation.contains_key(&'A'));
15776 assert!(!saved.candidates[0].empty);
15777 assert_eq!(
15778 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15779 "retain implementation work\n"
15780 );
15781 std::fs::remove_file(lock).unwrap();
15782 runner.state = saved;
15783 } else {
15784 assert_eq!(runner.state.status, RunStatus::Blocked);
15785 assert_eq!(runner.state.candidates[0].commits, 1);
15786 }
15787 runner.roles.implementers.clear();
15790 runner.state.config.disk.min_free_bytes = 0;
15791 runner.state.resource_block = None;
15792 runner.implement().await.unwrap();
15793 assert!(runner.state.candidate_preservation.is_empty());
15794 assert_eq!(runner.state.candidates[0].commits, 1);
15795 assert!(!runner.state.candidates[0].empty);
15796 assert!(git::is_clean(&wt).await.unwrap());
15797 assert!(
15798 crate::run::read_artifact(&runner.state, "cand-A.patch")
15799 .unwrap()
15800 .contains("retain implementation work")
15801 );
15802 }
15803 }
15804
15805 #[tokio::test]
15806 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15807 let (_tmp, mut runner) = preservation_resume_fixture().await;
15808 runner.implement().await.unwrap();
15810 assert!(runner.state.candidate_preservation.is_empty());
15811 assert!(
15812 !runner
15813 .state
15814 .dir()
15815 .join("artifacts/candidate-A-recovery")
15816 .exists()
15817 );
15818 runner.fold_losers().await.unwrap();
15819 assert!(!runner.state.candidates[0].worktree.exists());
15820 for schema in [14, crate::run::SCHEMA] {
15821 runner.state.schema = schema;
15822 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15823 runner.state.save().unwrap();
15824 runner.state = RunState::load(&runner.state.id).unwrap();
15825 runner.implement().await.unwrap();
15826 assert!(runner.state.candidate_preservation.is_empty());
15827 assert!(runner.state.candidates[0].folded);
15828 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15829 }
15830 }
15831
15832 #[tokio::test]
15833 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15834 let (_tmp, mut runner) = preservation_resume_fixture().await;
15835 runner.state.candidates.retain(|c| c.label == 'A');
15836 runner.state.tally = None;
15837 runner.state.judge_skipped = false;
15838 let candidate = &mut runner.state.candidates[0];
15839 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15840 candidate.failed = Some("deployment validation unavailable".into());
15841 std::fs::write(
15842 candidate.worktree.join("repair"),
15843 "future launcher policy\n",
15844 )
15845 .unwrap();
15846 runner
15847 .state
15848 .candidate_preservation
15849 .insert('A', "awaiting persistence".into());
15850 runner.persist_candidates().await.unwrap();
15851 let candidate = &runner.state.candidates[0];
15852 assert!(!candidate.empty);
15853 assert_eq!(candidate.commits, 1);
15854 assert_eq!(candidate.files, 1);
15855 assert!(candidate.failed.is_none());
15856 assert!(candidate.summary.contains("EPERM"));
15857 assert!(
15858 crate::run::read_artifact(&runner.state, "cand-A.patch")
15859 .unwrap()
15860 .contains("future launcher policy")
15861 );
15862 assert!(runner.state.candidate_preservation.is_empty());
15863 }
15864
15865 #[tokio::test]
15866 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15867 let (_tmp, mut runner) = preservation_resume_fixture().await;
15868 let wt = runner.state.candidates[0].worktree.clone();
15869 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15870 runner.state.schema = 14;
15871 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15872 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15873 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15874 for _ in 0..2 {
15875 assert!(runner.implement().await.is_err());
15876 assert_eq!(runner.state.status, RunStatus::Blocked);
15877 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15878 assert!(
15879 runner.state.candidates[0].empty,
15880 "old verdict must remain unusable"
15881 );
15882 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15883 assert!(runner.state.judge_skipped);
15884 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15885 assert!(
15886 crate::run::read_artifact(&runner.state, "cand-A.patch")
15887 .unwrap()
15888 .contains("owned edits")
15889 );
15890 assert!(wt.exists());
15891 assert_eq!(
15892 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15893 .await
15894 .unwrap(),
15895 1
15896 );
15897 runner.state = RunState::load(&runner.state.id).unwrap();
15898 }
15899 }
15900
15901 #[tokio::test]
15902 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15903 let (_tmp, mut runner) = preservation_resume_fixture().await;
15904 let wt = runner.state.candidates[0].worktree.clone();
15905 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15906 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15907 let index_lock = PathBuf::from(
15908 git::git(
15909 &wt,
15910 &[
15911 "rev-parse",
15912 "--path-format=absolute",
15913 "--git-path",
15914 "index.lock",
15915 ],
15916 )
15917 .await
15918 .unwrap(),
15919 );
15920 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15921 runner.state.schema = 14;
15922 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15923 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15924 assert!(runner.implement().await.is_err());
15925 assert!(runner.after_implement().is_err());
15926 assert!(runner.fold_losers().await.is_err());
15927 assert_eq!(
15928 std::fs::read_to_string(wt.join("recovered")).unwrap(),
15929 "failed seat edits"
15930 );
15931 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15932 runner.state = RunState::load(&runner.state.id).unwrap();
15933 std::fs::remove_file(index_lock).unwrap();
15934 assert!(runner.implement().await.is_err());
15936 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15937 assert!(runner.state.candidates[0].empty);
15938 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15939 assert!(
15940 crate::run::read_artifact(&runner.state, "cand-A.patch")
15941 .unwrap()
15942 .contains("failed seat edits")
15943 );
15944 assert!(wt.exists());
15945 }
15946
15947 #[tokio::test]
15948 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15949 let (_tmp, mut runner) = preservation_resume_fixture().await;
15950 let repo = runner.state.repo.clone();
15951 let common = git::git(
15952 &repo,
15953 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15954 )
15955 .await
15956 .unwrap();
15957 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15958 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15959 assert!(runner.fold_losers().await.is_err());
15960 assert!(!runner.state.candidates[0].worktree.exists());
15961 assert!(!runner.state.candidates[0].folded);
15962 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15963 runner.state = RunState::load(&runner.state.id).unwrap();
15964 std::fs::remove_file(branch_lock).unwrap();
15965 let judge = runner.state.worktree_root().join("judge-1");
15966 git::worktree_add_detached(&repo, &judge, "main")
15967 .await
15968 .unwrap();
15969 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15970 runner.roles.judges = vec![spec("judge")];
15971 assert!(runner.fold_losers().await.is_err());
15972 runner.state = RunState::load(&runner.state.id).unwrap();
15973 assert!(
15974 runner.state.candidates[0].folded,
15975 "candidate progress saved before judge failure"
15976 );
15977 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15978 assert!(judge.join("uncommitted").exists());
15979 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15980 runner.fold_losers().await.unwrap();
15981 assert!(!judge.exists());
15982 runner.fold_losers().await.unwrap();
15983 }
15984
15985 #[tokio::test]
15986 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15987 let (_tmp, mut runner) = preservation_resume_fixture().await;
15988 runner.state.candidates.retain(|c| c.label == 'B');
15989 runner.state.tally = None;
15990 runner.state.judge_skipped = false;
15991 runner.state.prep_complete = Some(true);
15992 runner.state.config.graph.advise = false;
15993 runner.state.config.graph.selected_implementer = Some("alpha".into());
15994 let wt = runner.state.candidates[0].worktree.clone();
15995 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15996 runner
15997 .state
15998 .candidate_preservation
15999 .insert('B', "awaiting persistence".into());
16000 let index_lock = git::git(
16001 &wt,
16002 &[
16003 "rev-parse",
16004 "--path-format=absolute",
16005 "--git-path",
16006 "index.lock",
16007 ],
16008 )
16009 .await
16010 .unwrap();
16011 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16012 runner.state.save().unwrap();
16013 assert!(runner.execute_graph().await.is_err());
16014 assert_eq!(runner.state.status, RunStatus::Blocked);
16015 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16016 std::fs::remove_file(index_lock).unwrap();
16017 for _ in 0..2 {
16020 runner.state = RunState::load(&runner.state.id).unwrap();
16021 runner.pause.park();
16022 runner.execute_graph().await.unwrap();
16023 assert_eq!(runner.state.status, RunStatus::Implementing);
16024 assert!(runner.state.candidate_preservation.is_empty());
16025 assert_eq!(runner.state.candidates[0].commits, 2);
16026 assert!(runner.state.tally.is_none());
16027 }
16028 runner.implement().await.unwrap();
16029 assert_eq!(runner.state.status, RunStatus::Judging);
16030 runner.state.status = RunStatus::Blocked;
16032 runner.state.candidates[0].failed = Some("implementation failed".into());
16033 runner.state.save().unwrap();
16034 let error = runner.execute_graph().await.unwrap_err();
16035 assert!(
16036 error
16037 .to_string()
16038 .contains("selected implementation is held")
16039 );
16040 }
16041
16042 #[tokio::test]
16043 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16044 let (_tmp, mut runner) = preservation_resume_fixture().await;
16045 let repo = runner.state.repo.clone();
16046 let candidate = runner.state.candidates[0].clone();
16047 let registered = git::worktree_holding(&repo, &candidate.branch)
16050 .await
16051 .unwrap()
16052 .unwrap();
16053 assert_eq!(
16054 registered.canonicalize().unwrap(),
16055 candidate.worktree.canonicalize().unwrap()
16056 );
16057 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16058 assert_eq!(
16059 git::worktree_holding(&repo, &candidate.branch)
16060 .await
16061 .unwrap(),
16062 Some(registered)
16063 );
16064 assert!(
16065 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16066 "the stale registration initially keeps the branch checked out"
16067 );
16068 runner.fold_losers().await.unwrap();
16069 assert!(runner.state.candidates[0].folded);
16070 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16071 assert_eq!(
16072 git::worktree_holding(&repo, &candidate.branch)
16073 .await
16074 .unwrap(),
16075 None
16076 );
16077 runner.state = RunState::load(&runner.state.id).unwrap();
16078 runner.fold_losers().await.unwrap();
16079 }
16080
16081 #[tokio::test]
16082 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16083 let (_tmp, mut runner) = preservation_resume_fixture().await;
16084 let repo = runner.state.repo.clone();
16085 let candidate = runner.state.candidates[0].clone();
16086 let winner = runner.state.candidates[1].clone();
16087 let registered = git::worktree_holding(&repo, &candidate.branch)
16088 .await
16089 .unwrap()
16090 .unwrap();
16091 assert_eq!(
16092 registered.canonicalize().unwrap(),
16093 candidate.worktree.canonicalize().unwrap()
16094 );
16095 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16096 assert_eq!(
16097 git::worktree_holding(&repo, &candidate.branch)
16098 .await
16099 .unwrap(),
16100 Some(registered)
16101 );
16102 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16103
16104 for _ in 0..2 {
16105 fold_run(&mut runner.state, false, &crate::run::home())
16106 .await
16107 .unwrap();
16108 runner.state = RunState::load(&runner.state.id).unwrap();
16109 assert!(runner.state.candidates[0].folded);
16110 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16111 assert_eq!(
16112 git::worktree_holding(&repo, &candidate.branch)
16113 .await
16114 .unwrap(),
16115 None
16116 );
16117 assert!(!runner.state.candidates[1].folded);
16118 assert!(winner.worktree.exists());
16119 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16120 }
16121 }
16122
16123 #[tokio::test]
16124 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16125 ask_test_home();
16126 let tmp = tempfile::tempdir().unwrap();
16127 let repo = tmp.path().join("repo");
16128 std::fs::create_dir(&repo).unwrap();
16129 init_repo(&repo);
16130 let mut runner = runner_at(RunStatus::Implementing);
16131 runner.state.repo = repo.clone();
16132 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16133 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16134 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16135 let branch = format!("candidate-{label}");
16136 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16137 git::worktree_add_branch(&repo, &wt, &branch, "main")
16138 .await
16139 .unwrap();
16140 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16141 git::git(&wt, &["add", "README.md"]).await.unwrap();
16142 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16143 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16144 runner.state.candidates.push(Candidate {
16145 index,
16146 label,
16147 agent: "alpha".to_owned(),
16148 branch,
16149 worktree: wt,
16150 summary: "completed implementation".to_owned(),
16151 stat: String::new(),
16152 files: 0,
16153 commits: 0,
16154 empty: false,
16155 failed: None,
16156 verified_noop: None,
16157 duration_ms: 42,
16158 folded: false,
16159 });
16160 runner
16161 .state
16162 .candidate_preservation
16163 .insert(label, "awaiting persistence".to_owned());
16164 }
16165 let wt = runner.state.candidates[0].worktree.clone();
16166 let index_lock = git::git(
16167 &wt,
16168 &[
16169 "rev-parse",
16170 "--path-format=absolute",
16171 "--git-path",
16172 "index.lock",
16173 ],
16174 )
16175 .await
16176 .unwrap();
16177 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16178 runner.state.save().unwrap();
16179 assert!(runner.persist_candidates().await.is_err());
16180 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16181 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16182 assert_eq!(
16183 runner.state.candidates[1].commits, 1,
16184 "other results are still collected"
16185 );
16186 assert!(
16187 !runner.state.candidates[0].empty,
16188 "failed rescue is not no change"
16189 );
16190 assert_eq!(
16191 runner.state.candidates[0].summary,
16192 "completed implementation"
16193 );
16194 assert!(runner.after_implement().is_err());
16195 let home = tmp.path().join("fold-home");
16196 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16197 assert!(wt.join("new-file").exists());
16198 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16199 assert!(recovery.join("staged.patch").exists());
16200 assert!(recovery.join("unstaged.patch").exists());
16201 assert_eq!(
16202 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16203 "untracked edits\n"
16204 );
16205 runner.state.begin_pass();
16206 runner.state.end_pass_parked(ParkCause::Upgrade {
16207 from: "1.0".into(),
16208 to: Some("1.1".into()),
16209 });
16210 runner.state.save().unwrap();
16211 runner.state = RunState::load(&runner.state.id).unwrap();
16212 assert!(matches!(
16213 runner.state.passes.last().unwrap().end,
16214 Some(crate::run::PassEnd::Parked {
16215 cause: ParkCause::Upgrade { .. }
16216 })
16217 ));
16218 std::fs::remove_file(index_lock).unwrap();
16219 runner.implement().await.unwrap();
16221 assert!(runner.state.candidate_preservation.is_empty());
16222 assert!(
16223 runner
16224 .state
16225 .candidates
16226 .iter()
16227 .all(|c| c.commits == 1 && !c.empty)
16228 );
16229 assert!(git::is_clean(&wt).await.unwrap());
16230 assert!(
16231 crate::run::read_artifact(&runner.state, "cand-A.patch")
16232 .unwrap()
16233 .contains("later tracked edits")
16234 );
16235 }
16236
16237 #[tokio::test]
16238 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16239 ask_test_home();
16240 let tmp = tempfile::tempdir().unwrap();
16241 let repo = tmp.path().join("repo");
16242 std::fs::create_dir(&repo).unwrap();
16243 init_repo(&repo);
16244 let wt = tmp.path().join("candidate");
16245 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16246 .await
16247 .unwrap();
16248 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16249 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16250 assert!(wt.join("uncommitted").exists());
16251 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16252 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16253 let mut runner = runner_at(RunStatus::Implementing);
16254 runner.state.repo = repo.clone();
16255 runner.state.base_commit = "missing-base".to_owned();
16256 runner.state.candidates.push(Candidate {
16257 index: 0,
16258 label: 'A',
16259 agent: "alpha".into(),
16260 branch: "candidate".into(),
16261 worktree: wt.clone(),
16262 summary: "completed".into(),
16263 stat: String::new(),
16264 files: 0,
16265 commits: 0,
16266 empty: false,
16267 failed: None,
16268 verified_noop: None,
16269 duration_ms: 1,
16270 folded: false,
16271 });
16272 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16273 assert!(
16274 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16275 .await
16276 .is_err()
16277 );
16278 assert!(!runner.state.candidates[0].folded);
16279 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16280 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16281 runner
16282 .state
16283 .candidate_preservation
16284 .insert('A', "pending".into());
16285 assert!(runner.persist_candidates().await.is_err());
16286 assert!(!runner.state.candidates[0].empty);
16287 assert!(!runner.state.candidates[0].folded);
16288 assert!(wt.exists());
16289 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16290 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16291 std::fs::create_dir_all(&patch_path).unwrap();
16292 assert!(
16293 runner.persist_candidates().await.is_err(),
16294 "artifact failure stays pending"
16295 );
16296 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16297 assert!(!runner.state.candidates[0].empty);
16298 std::fs::remove_dir(patch_path).unwrap();
16299 runner.persist_candidates().await.unwrap();
16300 assert!(
16301 runner.state.candidates[0].empty,
16302 "only successful inspection confirms no change"
16303 );
16304 }
16305
16306 fn init_repo(dir: &Path) {
16307 let run = |args: &[&str]| {
16308 let out = std::process::Command::new("git")
16309 .args(args)
16310 .current_dir(dir)
16311 .quiet()
16312 .output()
16313 .expect("spawn git");
16314 assert!(
16315 out.status.success(),
16316 "git {args:?} failed: {}",
16317 String::from_utf8_lossy(&out.stderr)
16318 );
16319 };
16320 run(&["init", "-b", "main"]);
16321 run(&["config", "user.name", "magi test"]);
16322 run(&["config", "user.email", "magi@example.com"]);
16323 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16324 run(&["add", "-A"]);
16325 run(&["commit", "-m", "init"]);
16326 }
16327
16328 fn ask_test_home() {
16336 crate::run::pin_test_home();
16337 }
16338
16339 #[test]
16340 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16341 ask_test_home();
16342 let mut runner = runner_at(RunStatus::Prep);
16343 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16344 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16347 let error = guard
16348 .finish(
16349 &mut runner.state,
16350 Err(
16351 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16352 .into(),
16353 ),
16354 )
16355 .unwrap_err();
16356 assert_eq!(
16357 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16358 std::io::ErrorKind::PermissionDenied
16359 );
16360 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16361 let message = format!("{error:#}");
16362 assert!(message.contains("original refusal"));
16363 assert!(!message.contains("could not save terminal"));
16364 assert_eq!(
16365 error.chain().count(),
16366 2,
16367 "only startup attribution wraps the original error"
16368 );
16369 let saved = RunState::load(&runner.state.id).unwrap();
16370 assert_eq!(saved.status, RunStatus::Blocked);
16371 assert!(!saved.startup.unwrap().complete);
16372 }
16373
16374 #[test]
16375 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16376 ask_test_home();
16377 let mut runner = runner_at(RunStatus::Prep);
16378 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16379 let id = runner.state.id.clone();
16380 drop(guard);
16381 let saved = RunState::load(&id).unwrap();
16382 assert_eq!(saved.status, RunStatus::Blocked);
16383 assert!(!saved.startup.as_ref().unwrap().complete);
16384 assert!(
16385 saved
16386 .startup
16387 .unwrap()
16388 .failure
16389 .unwrap()
16390 .contains("interrupted")
16391 );
16392 assert!(Runner::resume(&id).is_err());
16393 }
16394
16395 #[test]
16396 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16397 ask_test_home();
16398 let mut runner = runner_at(RunStatus::Prep);
16399 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16400 let error = guard
16401 .finish(
16402 &mut runner.state,
16403 Err(
16404 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16405 .into(),
16406 ),
16407 )
16408 .unwrap_err();
16409 assert_eq!(
16410 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16411 std::io::ErrorKind::PermissionDenied
16412 );
16413 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16414 let mut task = crate::queue::Task::new(
16415 "review".into(),
16416 "criteria".into(),
16417 runner.state.repo.clone(),
16418 crate::queue::Source::Human,
16419 );
16420 task.record_startup_failure(failure.clone());
16421 task.record_startup_failure(failure.clone());
16422 assert_eq!(task.startup_failures.len(), 1);
16423 assert!(task.runs.is_empty());
16424 assert_eq!(task.attempts, 0);
16425 }
16426
16427 #[cfg(unix)]
16428 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16429 ask_test_home();
16430 let tmp = tempfile::tempdir().unwrap();
16431 let repo = tmp.path().join("repo");
16432 std::fs::create_dir(&repo).unwrap();
16433 init_repo(&repo);
16434 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16435 let origin = tmp.path().join("origin.git");
16436 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16437 .await
16438 .unwrap();
16439 git::git(
16440 &repo,
16441 &["remote", "add", "origin", origin.to_str().unwrap()],
16442 )
16443 .await
16444 .unwrap();
16445 git::git(&repo, &["checkout", "-b", "fix/retained"])
16446 .await
16447 .unwrap();
16448 std::fs::write(repo.join("retained"), "retained change").unwrap();
16449 git::git(&repo, &["add", "."]).await.unwrap();
16450 git::git(&repo, &["commit", "-m", "retained implementation"])
16451 .await
16452 .unwrap();
16453 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16454 git::git(&repo, &["checkout", "main"]).await.unwrap();
16455 std::fs::write(repo.join("base-change"), "new base").unwrap();
16456 git::git(&repo, &["add", "."]).await.unwrap();
16457 git::git(&repo, &["commit", "-m", "base moved"])
16458 .await
16459 .unwrap();
16460 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16461 git::git(&repo, &["checkout", "fix/retained"])
16462 .await
16463 .unwrap();
16464 let mut runner = runner_at(RunStatus::Voting);
16465 runner.state.repo = repo.clone();
16466 runner.state.base_commit = base;
16467 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16468 runner.state.config.disk.min_free_bytes = 0;
16469 runner.state.config.graph.review_rounds = 1;
16470 runner.state.config.verify.e2e = vec!["true".into()];
16471 runner.state.config.verify.gate = vec!["true".into()];
16472 runner.state.candidates = vec![Candidate {
16473 index: 0,
16474 label: 'A',
16475 agent: "fixture".into(),
16476 branch: "fix/retained".into(),
16477 worktree: repo.clone(),
16478 summary: String::new(),
16479 stat: String::new(),
16480 files: 1,
16481 commits: 1,
16482 empty: false,
16483 failed: None,
16484 verified_noop: None,
16485 duration_ms: 0,
16486 folded: false,
16487 }];
16488 runner.state.tally = Some(Tally {
16489 first_choice: BTreeMap::new(),
16490 borda: BTreeMap::new(),
16491 winner: 'A',
16492 rankings: 0,
16493 unanimous_initial: true,
16494 deliberated: false,
16495 changed_votes: 0,
16496 unanimous_final: true,
16497 tie_break: None,
16498 judges: 0,
16499 present: 0,
16500 quorum: 0,
16501 met_quorum: true,
16502 uncontested: Some("review-only".into()),
16503 });
16504 let mut task = crate::queue::Task::new(
16505 "fixture".into(),
16506 "fixture".into(),
16507 repo,
16508 crate::queue::Source::Human,
16509 );
16510 task.link_run(&runner.state.id);
16511 crate::queue::Queue::open().put(&mut task).unwrap();
16512 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16513 runner.state.base_sync = Some(BaseSync {
16514 tip: String::new(),
16515 behind: 1,
16516 attempts: BASE_SYNC_ROUNDS,
16517 conflict: None,
16518 already_in: None,
16519 });
16520 runner.sync_to_base().await.unwrap();
16521 assert_eq!(runner.state.status, RunStatus::Blocked);
16522 (tmp, runner, head, task.id)
16523 }
16524
16525 #[cfg(unix)]
16526 #[tokio::test]
16527 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16528 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16529 if std::env::var_os(CHILD).is_none() {
16530 let home = tempfile::tempdir().unwrap();
16531 let output = std::process::Command::new(std::env::current_exe().unwrap())
16532 .args([
16533 "--exact",
16534 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16535 "--nocapture",
16536 ])
16537 .env(CHILD, "1")
16538 .env("MAGI_HOME", home.path())
16539 .env("MAGI_WORKER_HOME", home.path())
16540 .env("MAGI_WORKER_ID", "abcdef")
16541 .output()
16542 .unwrap();
16543 assert!(
16544 output.status.success()
16545 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16546 "{}\n{}",
16547 String::from_utf8_lossy(&output.stdout),
16548 String::from_utf8_lossy(&output.stderr)
16549 );
16550 return;
16551 }
16552 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16553 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16554 let home = crate::run::home();
16555 let queue = crate::queue::Queue::open();
16556 let mut owner = queue.get(&task).unwrap();
16557 owner.release_operator("test-release", None);
16558 owner.start(runner.state.id.clone());
16559 queue.put(&mut owner).unwrap();
16560 runner.state.driver_exited = true;
16561 runner.state.task_instruction = Some(owner.instruction.clone());
16562 runner.state.save().unwrap();
16563 let _claim = queue.claim(&task).unwrap();
16564 let reserve = |id: &str| {
16565 let directory = home.join("workers/jobs").join(id);
16566 std::fs::create_dir_all(&directory).unwrap();
16567 let job = crate::worker::Job {
16568 id: id.into(),
16569 sequence: 1,
16570 kind: "task".into(),
16571 key: format!("task:{task}"),
16572 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16573 generation: "fixture-generation".into(),
16574 version: None,
16575 status: crate::worker::Status::Running,
16576 result: None,
16577 pid: Some(std::process::id()),
16578 created_at: 0,
16579 finished_at: None,
16580 };
16581 std::fs::write(
16582 directory.join("job.json"),
16583 serde_json::to_vec(&job).unwrap(),
16584 )
16585 .unwrap();
16586 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16587 };
16588 let _own = reserve("abcdef");
16589 let other = reserve("fedcba");
16590 assert!(
16591 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16592 .await
16593 .is_err()
16594 );
16595 assert!(
16596 RunState::load(&runner.state.id)
16597 .unwrap()
16598 .base_sync_recovery
16599 .authorizations
16600 .is_empty()
16601 );
16602 drop(other);
16603 assert!(crate::daemon::is_working_on(
16604 &home,
16605 &runner.state.id,
16606 Timestamp::now()
16607 ));
16608 assert!(
16609 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16610 );
16611 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16612 .await
16613 .unwrap();
16614 assert!(
16615 RunState::load(&runner.state.id)
16616 .unwrap()
16617 .base_recovery_pending()
16618 );
16619 }
16620
16621 #[cfg(unix)]
16622 #[tokio::test]
16623 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16624 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16625 let home = crate::run::home();
16626 runner.state.driver_pid = Some(std::process::id());
16627 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16628 runner.state.driver_exited = false;
16629 runner.state.begin_pass();
16630 runner.state.save().unwrap();
16631 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16632 driver.record(&runner.state).unwrap();
16633 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16634 drop(driver);
16636 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16637 assert_eq!(
16638 runner.state.liveness_under(&home, false),
16639 crate::run::Liveness::Dead
16640 );
16641 let queue = crate::queue::Queue::open();
16642 let _claim = queue.claim(&task).unwrap();
16643 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16644 .await
16645 .unwrap();
16646 let disk = RunState::load(&runner.state.id).unwrap();
16647 assert!(!disk.driver_exited);
16648 assert!(disk.base_recovery_pending());
16649 }
16650
16651 #[cfg(unix)]
16652 #[tokio::test]
16653 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16654 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16655 if std::env::var_os(CHILD).is_none() {
16656 use std::os::unix::fs::PermissionsExt;
16657 let tmp = tempfile::tempdir().unwrap();
16658 let gh = tmp.path().join("gh");
16659 std::fs::write(&gh, r#"#!/bin/sh
16660case "$1 $2" in
16661 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16662 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16663 *) exit 2 ;;
16664esac
16665"#).unwrap();
16666 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16667 let mut paths = vec![tmp.path().to_path_buf()];
16668 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16669 let output = std::process::Command::new(std::env::current_exe().unwrap())
16670 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16671 .env(CHILD, "1")
16672 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16673 .env("PATH", std::env::join_paths(paths).unwrap())
16674 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16675 .output().unwrap();
16676 assert!(
16677 output.status.success()
16678 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16679 "{}\n{}",
16680 String::from_utf8_lossy(&output.stdout),
16681 String::from_utf8_lossy(&output.stderr)
16682 );
16683 return;
16684 }
16685 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16686 let repo = runner.state.repo.clone();
16687 git::git(&repo, &["checkout", "main"]).await.unwrap();
16688 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16689 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16690 git::git(&repo, &["checkout", "fix/retained"])
16691 .await
16692 .unwrap();
16693 runner.state.pr = Some(crate::run::PrRecord {
16694 url: "https://example.test/project/pull/1".into(),
16695 number: 1,
16696 state: "open".into(),
16697 checks: "unknown".into(),
16698 round: 0,
16699 rounds: 1,
16700 red_at_merge: Vec::new(),
16701 });
16702 runner.sync_to_base().await.unwrap();
16703 assert_eq!(runner.state.status, RunStatus::Blocked);
16704 assert_eq!(
16705 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16706 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16707 );
16708 assert!(runner.state.base_sync_closure_pending());
16709 assert!(crate::handover::can_resume(&runner.state));
16710 let queue = crate::queue::Queue::open();
16711 assert!(!crate::base_sync_recovery::held_task(
16712 &queue,
16713 &queue.get(&task).unwrap()
16714 ));
16715 assert!(runner.state.authorize_base_sync(&task).is_err());
16716 runner.state.base_sync_recovery.stop = None;
16718 runner.state.status = RunStatus::Voting;
16719 runner.state.gate_ran = true;
16720 runner.state.config.graph.review_rounds = 0;
16721 runner.state.save().unwrap();
16722 assert!(crate::handover::can_resume(&runner.state));
16723 runner.state = RunState::load(&runner.state.id).unwrap();
16724 runner.execute_graph().await.unwrap();
16725 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16726 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16727 assert_eq!(
16728 runner.state.base_sync.as_ref().unwrap().attempts,
16729 BASE_SYNC_ROUNDS
16730 );
16731 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16732 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16733 assert!(runner.state.reviews.is_empty());
16734 assert!(runner.state.gate.is_empty());
16735 }
16736
16737 #[cfg(unix)]
16738 #[tokio::test]
16739 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16740 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16741 runner.state.driver_exited = true;
16742 runner.state.save().unwrap();
16743 let home = crate::run::home();
16744 let queue = crate::queue::Queue::open();
16745 let _claim = queue.claim(&task).unwrap();
16746 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16747 assert!(
16748 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16749 .await
16750 .is_err()
16751 );
16752 drop(lease);
16753 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16754 assert!(
16755 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16756 .await
16757 .is_err()
16758 );
16759 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16760 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16761 .await
16762 .unwrap();
16763 let disk = RunState::load(&runner.state.id).unwrap();
16764 assert!(disk.base_recovery_pending());
16765 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16766 assert!(
16767 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16768 .await
16769 .is_err()
16770 );
16771 assert_eq!(
16772 RunState::load(&runner.state.id)
16773 .unwrap()
16774 .base_sync_recovery
16775 .authorizations
16776 .len(),
16777 1
16778 );
16779 let mut owner = queue.get(&task).unwrap();
16780 owner.runs.push("20260101-000000-ffff".into());
16781 queue.put(&mut owner).unwrap();
16782 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16783 .await
16784 .unwrap_err();
16785 assert!(error.to_string().contains("run task ownership changed"));
16786 }
16787
16788 #[cfg(unix)]
16789 #[tokio::test]
16790 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16791 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16792 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16793 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16794 legacy["status"] = serde_json::json!("voting");
16795 runner.state = serde_json::from_value(legacy).unwrap();
16796 runner.state.save().unwrap();
16797 for _ in 0..3 {
16798 runner.execute_graph().await.unwrap();
16799 assert_eq!(runner.state.status, RunStatus::Blocked);
16800 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16801 assert!(!crate::handover::can_resume(&runner.state));
16802 assert_eq!(
16803 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16804 head
16805 );
16806 assert!(runner.state.reviews.is_empty());
16807 assert!(!runner.state.gate_ran);
16808 }
16809 let notice = crate::notices::run_ended(&runner.state).unwrap();
16810 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16811 runner.state.begin_pass();
16812 runner.state.status = RunStatus::Voting;
16813 runner.deliberate().await.unwrap();
16814 runner.vote().await.unwrap();
16815 runner.finish_after_tally().await.unwrap();
16816 assert_eq!(runner.state.status, RunStatus::Blocked);
16817 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16818 assert_eq!(
16819 crate::notices::run_ended(&runner.state).unwrap().since,
16820 notice.since
16821 );
16822 }
16823
16824 #[cfg(unix)]
16825 #[tokio::test]
16826 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16827 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16828 for detail in [
16829 "fix/retained conflicts with origin/main and did not rebase: conflict",
16830 "rebased fix/retained locally but origin refused the push",
16831 "unknown failure",
16832 ] {
16833 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16834 runner.state.record_base_sync_stop(&head);
16835 assert!(runner.state.authorize_base_sync(&task).is_err());
16836 runner.state.status = RunStatus::Voting;
16837 runner.execute_graph().await.unwrap();
16838 assert_eq!(runner.state.status, RunStatus::Blocked);
16839 assert!(!crate::handover::can_resume(&runner.state));
16840 assert_eq!(
16841 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16842 head
16843 );
16844 }
16845 }
16846
16847 #[cfg(unix)]
16848 #[tokio::test]
16849 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16850 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16851 runner.state.authorize_base_sync(&task).unwrap();
16852 assert!(runner.state.consume_base_recovery(&head));
16853 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16854 runner.state.reserve_base_operation(&head, &tip);
16855 runner.state.save().unwrap();
16856 runner.state = RunState::load(&runner.state.id).unwrap();
16857 assert!(!runner.state.consume_base_recovery(&head));
16858 runner.sync_to_base().await.unwrap();
16859 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16860 assert_eq!(runner.state.base_episode_used(), 1);
16861 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16862 assert!(
16863 runner
16864 .state
16865 .base_sync_recovery
16866 .operations
16867 .last()
16868 .unwrap()
16869 .finished
16870 );
16871 assert!(!runner.state.base_sync_blocked());
16872 runner.sync_to_base().await.unwrap();
16873 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16874 }
16875
16876 #[cfg(unix)]
16877 #[tokio::test]
16878 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16879 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16880 let original = runner
16882 .state
16883 .base_sync
16884 .as_ref()
16885 .unwrap()
16886 .conflict
16887 .clone()
16888 .unwrap();
16889 runner.state.base_sync.as_mut().unwrap().conflict = Some(
16890 original
16891 .replace("synchronization(s)", "rebase(s)")
16892 .replace("synchronizing", "rebasing"),
16893 );
16894 runner.state.base_sync_recovery = Default::default();
16895 runner.state.record_base_sync_stop(&head);
16896 assert_eq!(
16897 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16898 crate::base_sync_recovery::StopKind::Exhausted
16899 );
16900 runner.state.authorize_base_sync(&task).unwrap();
16901 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16902 let auth = legacy["base_sync_recovery"]["authorizations"][0]
16903 .as_object_mut()
16904 .unwrap();
16905 auth.remove("interval");
16906 auth.remove("scope");
16907 runner.state = serde_json::from_value(legacy).unwrap();
16908 assert!(runner.state.consume_base_recovery(&head));
16909 assert_eq!(runner.state.base_episode_used(), 3);
16910 runner.sync_to_base().await.unwrap();
16911 assert_eq!(runner.state.base_episode_used(), 4);
16912 runner.state.resource_block = Some(crate::run::ResourceBlock {
16913 node: "gate".into(),
16914 kind: "cache".into(),
16915 detail: "legacy wait".into(),
16916 at: Timestamp::now(),
16917 cause: crate::run::ResourceBlockCause::Unrecorded,
16918 });
16919 runner.state.save().unwrap();
16920 assert!(
16921 runner
16922 .state
16923 .base_sync_recovery
16924 .episodes
16925 .last()
16926 .unwrap()
16927 .boundary
16928 .is_none()
16929 );
16930 runner.state.resource_block = None;
16931 advance_base_fixture(&runner).await;
16932 runner.sync_to_base().await.unwrap();
16933 assert!(runner.state.base_sync_blocked());
16934 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16935 }
16936
16937 #[cfg(unix)]
16938 async fn advance_base_fixture(runner: &Runner) {
16939 let repo = &runner.state.repo;
16940 for args in [
16941 vec!["checkout", "main"],
16942 vec!["commit", "--allow-empty", "-m", "base advanced"],
16943 vec!["push", "origin", "main"],
16944 vec!["checkout", "fix/retained"],
16945 ] {
16946 git::git(repo, &args).await.unwrap();
16947 }
16948 }
16949
16950 #[cfg(unix)]
16951 fn fresh_base_interval_fixture(runner: &mut Runner) {
16952 runner.state.base_sync.as_mut().unwrap().attempts = 0;
16953 runner.state.base_sync.as_mut().unwrap().conflict = None;
16954 runner.state.base_sync_recovery = Default::default();
16955 runner.state.status = RunStatus::Voting;
16956 }
16957
16958 #[cfg(unix)]
16959 #[tokio::test]
16960 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
16961 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
16962 fresh_base_interval_fixture(&mut runner);
16963 runner.state.begin_pass();
16964 runner.sync_to_base().await.unwrap();
16965 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16966 runner.state.resource_block = Some(crate::run::ResourceBlock {
16967 node: "gate".into(),
16968 kind: "build cache".into(),
16969 detail: "busy".into(),
16970 at: Timestamp::now(),
16971 cause: crate::run::ResourceBlockCause::Timeout {
16972 waited_secs: 1,
16973 limit_secs: 1,
16974 },
16975 });
16976 runner.state.save().unwrap();
16977 let queue = crate::queue::Queue::open();
16978 let mut owner = queue.get(&task).unwrap();
16979 runner.state.task_instruction = Some(owner.instruction.clone());
16980 owner.release_operator("fixture-owner", None);
16981 queue.put(&mut owner).unwrap();
16982 let approval = owner.resumption.clone().unwrap();
16983 runner.state.resource_block = None;
16984 runner.state.save().unwrap();
16985 runner.state = RunState::load(&runner.state.id).unwrap();
16986 advance_base_fixture(&runner).await;
16987 runner.sync_to_base().await.unwrap();
16988 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
16989 assert_eq!(runner.state.base_episode_used(), 1);
16990 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16991 assert!(
16992 runner.state.base_sync_recovery.episodes[0]
16993 .boundary
16994 .as_ref()
16995 .unwrap()
16996 .consumed
16997 );
16998 assert!(
16999 !runner
17000 .state
17001 .renew_base_episode(&task, &approval, &head, "base")
17002 );
17003 for _ in 0..3 {
17004 owner.release_operator("fixture-owner", None);
17005 queue.put(&mut owner).unwrap();
17006 runner.state = RunState::load(&runner.state.id).unwrap();
17007 runner.state.begin_pass();
17008 runner.sync_to_base().await.unwrap();
17009 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17010 assert_eq!(runner.state.base_episode_used(), 1);
17011 }
17012 runner.state.resource_block = Some(crate::run::ResourceBlock {
17013 node: "gate".into(),
17014 kind: "disk pressure".into(),
17015 detail: "free space required".into(),
17016 at: Timestamp::now(),
17017 cause: crate::run::ResourceBlockCause::Other,
17018 });
17019 runner.state.end_pass_parked(ParkCause::Interrupt {
17020 reason: "disk pressure".into(),
17021 });
17022 runner.state.save().unwrap();
17023 owner.release_operator("fixture-owner-after-disk", None);
17024 queue.put(&mut owner).unwrap();
17025 runner.state.resource_block = None;
17026 advance_base_fixture(&runner).await;
17027 runner.sync_to_base().await.unwrap();
17028 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17029 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17030 assert_eq!(runner.state.base_episode_used(), 1);
17031 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17032 }
17033
17034 #[cfg(unix)]
17035 #[tokio::test]
17036 async fn base_recovery_continuous_advancement_stops_at_four() {
17037 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17038 fresh_base_interval_fixture(&mut runner);
17039 for spent in 1..=BASE_SYNC_ROUNDS {
17040 runner.sync_to_base().await.unwrap();
17041 assert!(!runner.state.base_sync_blocked());
17042 assert_eq!(runner.state.base_episode_used(), spent);
17043 advance_base_fixture(&runner).await;
17044 }
17045 runner.sync_to_base().await.unwrap();
17046 assert_eq!(
17047 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17048 crate::base_sync_recovery::StopKind::Exhausted
17049 );
17050 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17051 assert!(stop.unverified_sync);
17052 let action = runner.state.base_sync_next_action();
17053 assert!(action.contains("never reached its gate"));
17054 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17055 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17056 runner.state = RunState::load(&runner.state.id).unwrap();
17057 runner.sync_to_base().await.unwrap();
17058 assert_eq!(runner.state.base_episode_used(), 4);
17059 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17060 }
17061
17062 #[cfg(unix)]
17063 #[tokio::test]
17064 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17065 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17066 runner.state.ensure_base_episode();
17067 runner.state.record_base_progress(&head);
17069 runner.state.base_sync_recovery.stop = None;
17070 runner.state.record_base_sync_stop(&head);
17071 assert!(
17072 !runner
17073 .state
17074 .base_sync_recovery
17075 .stop
17076 .as_ref()
17077 .unwrap()
17078 .unverified_sync
17079 );
17080
17081 runner.state.finish_base_operation(Some(&head));
17082 runner.state.gate_ran = true;
17084 runner.state.gate_head = Some("older-head".into());
17085 runner.state.base_sync_recovery.stop = None;
17086 runner.state.record_base_sync_stop(&head);
17087 runner.state.save().unwrap();
17088 runner.state = RunState::load(&runner.state.id).unwrap();
17089 assert!(
17090 runner
17091 .state
17092 .base_sync_recovery
17093 .stop
17094 .as_ref()
17095 .unwrap()
17096 .unverified_sync
17097 );
17098
17099 runner.state.gate_head = Some(head.clone());
17100 runner.state.base_sync_recovery.stop = None;
17101 runner.state.record_base_sync_stop(&head);
17102 assert!(
17103 !runner
17104 .state
17105 .base_sync_recovery
17106 .stop
17107 .as_ref()
17108 .unwrap()
17109 .unverified_sync
17110 );
17111 assert!(
17112 !runner
17113 .state
17114 .base_sync_next_action()
17115 .contains("never reached its gate")
17116 );
17117
17118 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17120 legacy["base_sync_recovery"]["stop"]
17121 .as_object_mut()
17122 .unwrap()
17123 .remove("unverified_sync");
17124 legacy["base_sync_recovery"]["episodes"][0]
17125 .as_object_mut()
17126 .unwrap()
17127 .remove("last_sync_head");
17128 runner.state = serde_json::from_value(legacy).unwrap();
17129 assert!(
17130 !runner
17131 .state
17132 .base_sync_recovery
17133 .stop
17134 .as_ref()
17135 .unwrap()
17136 .unverified_sync
17137 );
17138 assert!(
17139 runner.state.base_sync_recovery.episodes[0]
17140 .last_sync_head
17141 .is_none()
17142 );
17143 }
17144
17145 #[cfg(unix)]
17146 #[tokio::test]
17147 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17148 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17149 runner.state.authorize_base_sync(&task).unwrap();
17150 assert!(
17151 runner
17152 .state
17153 .base_sync_next_action()
17154 .contains("Approval recorded")
17155 );
17156 assert!(
17157 runner
17158 .state
17159 .base_sync_next_action()
17160 .contains("4/4 in episode 0, 4 lifetime")
17161 );
17162 assert!(runner.state.consume_base_recovery(&head));
17163 assert!(
17164 runner
17165 .state
17166 .base_sync_next_action()
17167 .contains("0/4 in episode 1, 4 lifetime")
17168 );
17169 }
17170
17171 #[cfg(unix)]
17172 #[tokio::test]
17173 async fn base_recovery_changed_head_invalidates_without_spending() {
17174 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17175 runner.state.authorize_base_sync(&task).unwrap();
17176 git::git(
17177 &runner.state.repo,
17178 &["commit", "--allow-empty", "-m", "external head change"],
17179 )
17180 .await
17181 .unwrap();
17182 runner.sync_to_base().await.unwrap();
17183 let a = runner
17184 .state
17185 .base_sync_recovery
17186 .authorizations
17187 .last()
17188 .unwrap();
17189 assert!(a.consumed);
17190 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17191 runner.state = RunState::load(&runner.state.id).unwrap();
17192 assert!(
17193 runner
17194 .state
17195 .base_sync_next_action()
17196 .contains("Approval invalidated")
17197 );
17198 assert!(!runner.state.base_recovery_pending());
17199 assert!(runner.state.base_sync_blocked());
17200 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17201 }
17202
17203 #[cfg(unix)]
17204 #[tokio::test]
17205 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17206 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17207 runner.state.driver_exited = true;
17208 runner.state.save().unwrap();
17209 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17210 .await
17211 .unwrap();
17212 runner.state = RunState::load(&runner.state.id).unwrap();
17213 advance_base_fixture(&runner).await;
17214 runner.sync_to_base().await.unwrap();
17215 let a = runner
17216 .state
17217 .base_sync_recovery
17218 .authorizations
17219 .last()
17220 .unwrap();
17221 assert!(
17222 a.consumed
17223 && a.invalidated
17224 .as_deref()
17225 .unwrap()
17226 .contains("fetched base changed")
17227 );
17228 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17229 assert_eq!(
17230 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17231 head
17232 );
17233 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17234 .await
17235 .unwrap();
17236 runner.state = RunState::load(&runner.state.id).unwrap();
17237 runner.sync_to_base().await.unwrap();
17238 assert!(!runner.state.base_sync_blocked());
17239 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17240 }
17241
17242 #[cfg(unix)]
17243 #[tokio::test]
17244 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17245 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17246 runner.state.authorize_base_sync(&task).unwrap();
17247 assert!(runner.state.consume_base_recovery(&head));
17248 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17249 runner.state.reserve_base_operation(&head, &tip);
17250 runner.state.save().unwrap();
17251 let scratch = runner.state.dir().join("base-sync");
17252 assert_eq!(
17253 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17254 .await
17255 .unwrap(),
17256 crate::rebase::Rebased::Applied
17257 );
17258 runner.state = RunState::load(&runner.state.id).unwrap();
17259 runner.sync_to_base().await.unwrap();
17260 assert!(
17261 runner
17262 .state
17263 .base_sync_recovery
17264 .operations
17265 .last()
17266 .unwrap()
17267 .finished
17268 );
17269 assert_eq!(runner.state.base_episode_used(), 1);
17270 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17271 assert!(!runner.state.base_sync_blocked());
17272 }
17273
17274 #[cfg(unix)]
17275 #[tokio::test]
17276 async fn base_recovery_crash_preserves_untracked_files() {
17277 for ignored in [false, true] {
17278 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17279 runner.state.authorize_base_sync(&task).unwrap();
17280 assert!(runner.state.consume_base_recovery(&head));
17281 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17282 runner.state.reserve_base_operation(&head, &tip);
17283 runner.state.save().unwrap();
17284 let scratch = runner.state.dir().join("base-sync");
17285 assert_eq!(
17286 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17287 .await
17288 .unwrap(),
17289 crate::rebase::Rebased::Applied
17290 );
17291 let repo = &runner.state.repo;
17292 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17293 assert_ne!(result, head);
17294 if ignored {
17295 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17296 .await
17297 .unwrap();
17298 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17299 }
17300 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17301 assert!(
17302 git::git_raw(repo, &["diff", "--quiet", &head])
17303 .await
17304 .unwrap()
17305 .ok()
17306 );
17307 runner.state = RunState::load(&runner.state.id).unwrap();
17309 runner.sync_to_base().await.unwrap();
17310 assert!(runner.state.base_sync_blocked());
17311 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17312 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17313 assert_eq!(
17314 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17315 result
17316 );
17317 assert_eq!(
17318 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17319 "retained work\n"
17320 );
17321 assert!(
17322 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17323 .await
17324 .unwrap()
17325 .ok()
17326 );
17327 }
17328 }
17329
17330 #[cfg(unix)]
17331 #[tokio::test]
17332 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17333 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17334 runner.state.authorize_base_sync(&task).unwrap();
17335 assert!(runner.state.consume_base_recovery(&head));
17336 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17337 runner.state.reserve_base_operation(&head, &tip);
17338 runner.state.save().unwrap();
17339 let scratch = runner.state.dir().join("base-sync");
17340 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17341 .await
17342 .unwrap();
17343 git::sync_to_head(&runner.state.repo).await.unwrap();
17344 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17345 git::git(&runner.state.repo, &["add", "foreign"])
17346 .await
17347 .unwrap();
17348 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17349 .await
17350 .unwrap();
17351 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17352 runner.state = RunState::load(&runner.state.id).unwrap();
17353 runner.sync_to_base().await.unwrap();
17354 assert!(runner.state.base_sync_blocked());
17355 assert!(
17356 !runner
17357 .state
17358 .base_sync_recovery
17359 .operations
17360 .last()
17361 .unwrap()
17362 .finished
17363 );
17364 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17365 assert_eq!(
17366 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17367 foreign_head
17368 );
17369 }
17370
17371 #[cfg(unix)]
17372 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17373 {
17374 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17375 fresh_base_interval_fixture(&mut runner);
17376 let repo = &runner.state.repo;
17377 git::git(repo, &["checkout", "main"]).await.unwrap();
17378 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17379 git::git(repo, &["add", "retained"]).await.unwrap();
17380 git::git(repo, &["commit", "-m", "upstream retained change"])
17381 .await
17382 .unwrap();
17383 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17384 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17385 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17386 let scratch = runner.state.dir().join("base-sync");
17387 assert!(matches!(
17388 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17389 .await
17390 .unwrap(),
17391 git::RebaseStart::Conflicted(_)
17392 ));
17393 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17394 agent: "fixture".into(),
17395 paths: vec!["retained".into()],
17396 from: Some(head.clone()),
17397 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17398 index_outside: None,
17399 finished: false,
17400 error: None,
17401 launcher: None,
17402 infra: None,
17403 outage: None,
17404 });
17405 (tmp, runner, head, tip, scratch)
17406 }
17407
17408 #[cfg(unix)]
17409 #[tokio::test]
17410 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17411 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17412 std::fs::write(
17415 scratch.join("retained"),
17416 "upstream and retained resolution\n",
17417 )
17418 .unwrap();
17419 runner.state.resource_block = Some(crate::run::ResourceBlock {
17420 node: "rebase".into(),
17421 kind: "repository coordination".into(),
17422 detail: "busy".into(),
17423 at: Timestamp::now(),
17424 cause: crate::run::ResourceBlockCause::Other,
17425 });
17426 runner.state.save().unwrap();
17427 let newer = git::git(
17430 &runner.state.repo,
17431 &[
17432 "commit-tree",
17433 "main^{tree}",
17434 "-p",
17435 "main",
17436 "-m",
17437 "base advanced during wait",
17438 ],
17439 )
17440 .await
17441 .unwrap();
17442 git::git(
17443 &runner.state.repo,
17444 &[
17445 "push",
17446 "origin",
17447 &format!("{}:refs/heads/main", newer.trim()),
17448 ],
17449 )
17450 .await
17451 .unwrap();
17452 runner.state = RunState::load(&runner.state.id).unwrap();
17453 runner.state.resource_block = None;
17454 runner.sync_to_base().await.unwrap();
17455 assert!(!runner.state.base_sync_blocked());
17456 let op = &runner.state.base_sync_recovery.operations[0];
17457 assert_eq!(op.head, head);
17458 assert_eq!(op.tip, tip);
17459 assert!(op.finished);
17460 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17461 assert_eq!(runner.state.rebase_fixes.len(), 1);
17462 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17463 .await
17464 .unwrap();
17465 assert_ne!(latest, tip);
17466 runner.state = RunState::load(&runner.state.id).unwrap();
17467 runner.sync_to_base().await.unwrap();
17468 assert!(!runner.state.base_sync_blocked());
17469 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17470 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17471 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17472 }
17473
17474 #[cfg(unix)]
17475 #[tokio::test]
17476 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17477 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17478 runner.state.reserve_base_operation(&head, &tip);
17479 runner.state.save().unwrap();
17480 std::fs::write(
17481 scratch.join("retained"),
17482 "upstream and retained resolution\n",
17483 )
17484 .unwrap();
17485 git::git(&scratch, &["add", "retained"]).await.unwrap();
17486 git::git(
17487 &scratch,
17488 &["-c", "core.editor=true", "rebase", "--continue"],
17489 )
17490 .await
17491 .unwrap();
17492 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17493 assert!(
17494 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17495 .await
17496 .unwrap()
17497 .is_empty()
17498 );
17499 assert_eq!(
17500 crate::rebase::finish(
17501 &mut runner.state,
17502 &scratch,
17503 "fix/retained",
17504 &head,
17505 &tip,
17506 &["retained".into()],
17507 ""
17508 )
17509 .await
17510 .unwrap(),
17511 crate::rebase::Rebased::Applied
17512 );
17513 assert!(!scratch.exists());
17514 runner.state = RunState::load(&runner.state.id).unwrap();
17516 let op = &runner.state.base_sync_recovery.operations[0];
17517 assert!(!op.finished);
17518 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17519 runner.sync_to_base().await.unwrap();
17520 assert!(!runner.state.base_sync_blocked());
17521 assert!(runner.state.base_sync_recovery.operations[0].finished);
17522 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17523 assert_eq!(runner.state.rebase_fixes.len(), 1);
17524 let winner = runner.state.winner().unwrap().clone();
17525 let wait = runner.repository_wait();
17526 assert!(
17527 verify_current_head(&mut runner.state, &winner, &wait)
17528 .await
17529 .unwrap()
17530 );
17531 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17532 }
17533
17534 #[cfg(unix)]
17535 #[tokio::test]
17536 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17537 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17538 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17539 runner.state.reserve_base_operation(&head, &tip);
17540 runner.state.save().unwrap();
17541 std::fs::write(
17542 scratch.join("retained"),
17543 "upstream and retained resolution\n",
17544 )
17545 .unwrap();
17546 git::git(&scratch, &["add", "retained"]).await.unwrap();
17547 git::git(
17548 &scratch,
17549 &["-c", "core.editor=true", "rebase", "--continue"],
17550 )
17551 .await
17552 .unwrap();
17553 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17554 assert!(!git::rebase_in_progress(&scratch).await);
17555 assert!(
17556 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17557 .await
17558 .unwrap()
17559 .is_empty()
17560 );
17561 match damage {
17562 Some("record") => {
17563 let path =
17564 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17565 .await
17566 .unwrap();
17567 std::fs::write(scratch.join(path), "").unwrap();
17568 }
17569 Some("foreign") => {
17570 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17571 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17572 git::git(&scratch, &["commit", "-m", "foreign work"])
17573 .await
17574 .unwrap();
17575 }
17576 Some("dirty") => {
17577 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17578 }
17579 _ => {}
17580 }
17581 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17582 runner.state = RunState::load(&runner.state.id).unwrap();
17584 assert!(
17585 runner.state.base_sync_recovery.operations[0]
17586 .validated_head
17587 .is_none()
17588 );
17589 runner.sync_to_base().await.unwrap();
17590 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17591 assert_eq!(runner.state.base_episode_used(), 1);
17592 assert_eq!(runner.state.rebase_fixes.len(), 1);
17593 assert_eq!(
17594 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17595 retained_head
17596 );
17597 if damage.is_some() {
17598 assert!(runner.state.base_sync_blocked());
17599 assert!(scratch.exists());
17600 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17601 assert!(
17602 runner.state.base_sync_recovery.operations[0]
17603 .validated_head
17604 .is_none()
17605 );
17606 } else {
17607 assert!(!runner.state.base_sync_blocked());
17608 assert!(!scratch.exists());
17609 assert!(runner.state.base_sync_recovery.operations[0].finished);
17610 assert_eq!(
17611 runner.state.base_sync_recovery.operations[0]
17612 .validated_head
17613 .as_deref(),
17614 Some(result.as_str())
17615 );
17616 let winner = runner.state.winner().unwrap().clone();
17617 let wait = runner.repository_wait();
17618 assert!(
17619 verify_current_head(&mut runner.state, &winner, &wait)
17620 .await
17621 .unwrap()
17622 );
17623 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17624 runner.state = RunState::load(&runner.state.id).unwrap();
17625 runner.sync_to_base().await.unwrap();
17626 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17627 }
17628 }
17629 }
17630
17631 #[cfg(unix)]
17632 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17633 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17634 runner.state.prep_complete = Some(true);
17636 runner.state.base_sync = None;
17637 runner.state.base_sync_recovery = Default::default();
17638 runner.state.config.graph.gate_fix_rounds = 1;
17639 let mut review = review_round(true, 0, 1, 1, false, true);
17640 review.head = head.clone();
17641 review.verified_head = Some(head.clone());
17642 runner.state.reviews = vec![review];
17643 runner.state.gate = vec![CommandOutcome {
17644 command: "old gate".into(),
17645 code: Some(1),
17646 output_tail: "old failure".into(),
17647 duration_ms: 0,
17648 resource_blocked: false,
17649 }];
17650 runner.state.gate_head = Some(head.clone());
17651 runner.state.gate_ran = true;
17652 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17653 agent: "fixture".into(),
17654 failed: runner.state.gate.clone(),
17655 notes: "spent".into(),
17656 committed: false,
17657 error: None,
17658 }];
17659 runner.state.land_approval = Some(crate::run::LandApproval {
17660 question: "old approval".into(),
17661 head: head.clone(),
17662 });
17663 runner.state.status = RunStatus::Blocked;
17664 (tmp, runner, head)
17665 }
17666
17667 #[cfg(unix)]
17668 #[tokio::test]
17669 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17670 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17671 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17672 let phases = tmp.path().join("phases");
17673 runner.state.config.verify.e2e = vec![format!(
17674 "echo e2e >> '{}'; {}",
17675 phases.display(),
17676 if e2e_ok {
17677 "true"
17678 } else {
17679 "echo new-e2e-failure; false"
17680 }
17681 )];
17682 runner.state.config.verify.gate = vec![format!(
17683 "echo gate >> '{}'; {}",
17684 phases.display(),
17685 if gate_ok {
17686 "true"
17687 } else {
17688 "echo new-gate-failure; false"
17689 }
17690 )];
17691 runner.state.save().unwrap();
17692 runner.state = RunState::load(&runner.state.id).unwrap();
17693 runner.sync_to_base().await.unwrap();
17694 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17695 assert_ne!(head, old);
17696 runner.state = RunState::load(&runner.state.id).unwrap();
17698 runner.execute_graph().await.unwrap();
17699 assert_eq!(runner.state.gate_fixes.len(), 1);
17700 assert_eq!(runner.state.reviews.len(), 1);
17701 assert_eq!(runner.state.reviews[0].head, old);
17702 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17703 assert_eq!(runner.state.verification_history.len(), 1);
17704 let history = &runner.state.verification_history[0];
17705 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17706 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17707 assert!(history.archived_at.is_some());
17708 assert_eq!(history.fixes_spent, Some(1));
17709 assert_eq!(history.gate[0].output_tail, "old failure");
17710 let archive_events: Vec<_> = runner
17711 .state
17712 .events
17713 .iter()
17714 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17715 .collect();
17716 assert_eq!(archive_events.len(), 1);
17717 assert!(archive_events[0].message.contains(&short(&old)));
17718 assert!(archive_events[0].message.contains(&short(&head)));
17719 assert!(
17720 archive_events[0]
17721 .message
17722 .contains("1 gate-fix round(s) already spent stay spent")
17723 );
17724 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17725 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17726 if e2e_ok {
17727 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17728 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17729 } else {
17730 assert!(!runner.state.gate_ran);
17731 }
17732 assert_eq!(
17733 runner.state.status,
17734 if e2e_ok && gate_ok {
17735 RunStatus::Ready
17736 } else {
17737 RunStatus::Blocked
17738 }
17739 );
17740 if !e2e_ok || !gate_ok {
17741 let cause = runner.state.verification_blocking_cause().unwrap();
17742 assert!(cause.contains(&head));
17743 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17744 runner.state.event("land", "rebased successfully");
17746 let notice = crate::notices::run_ended(&runner.state).unwrap();
17747 assert!(!format!("{notice:?}").contains("rebased successfully"));
17748 assert!(runner.state.merge.is_none());
17749 }
17750 let before = std::fs::read_to_string(&phases).unwrap();
17751 runner.state.save().unwrap();
17752 runner.state = RunState::load(&runner.state.id).unwrap();
17753 runner.review_loop().await.unwrap();
17754 runner.gate().await.unwrap();
17755 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17756 assert_eq!(runner.state.verification_history.len(), 1);
17757 let history = &runner.state.verification_history[0];
17758 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17759 assert!(history.archived_at.is_some());
17760 assert_eq!(history.fixes_spent, Some(1));
17761 }
17762 }
17763
17764 #[cfg(unix)]
17765 #[tokio::test]
17766 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17767 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17768 let marker = tmp.path().join("gate-ran");
17769 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17770 runner.review_loop().await.unwrap();
17771 runner.gate().await.unwrap();
17772 assert_eq!(runner.state.status, RunStatus::Blocked);
17773 assert!(!marker.exists());
17774 assert!(runner.state.verification_history.is_empty());
17775 runner.state.gate_head = None;
17776 runner.gate().await.unwrap();
17777 assert!(marker.exists());
17778 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17779 assert_eq!(runner.state.verification_history.len(), 1);
17780 assert_eq!(runner.state.gate_fixes.len(), 1);
17781 }
17782
17783 #[cfg(unix)]
17784 #[tokio::test]
17785 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17786 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17787 runner.sync_to_base().await.unwrap();
17788 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17789 runner.state.reconcile_verification_head(&head);
17790 for outcomes in [
17792 Vec::new(),
17793 vec![CommandOutcome {
17794 command: "true".into(),
17795 code: None,
17796 output_tail: "cache unavailable".into(),
17797 duration_ms: 0,
17798 resource_blocked: true,
17799 }],
17800 ] {
17801 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17802 head: head.clone(),
17803 passed: false,
17804 outcomes,
17805 });
17806 runner.state.gate_ran = false;
17807 runner.state.gate.clear();
17808 runner.state.save().unwrap();
17809 runner.state = RunState::load(&runner.state.id).unwrap();
17810 runner.merge().await.unwrap();
17811 assert!(runner.state.merge.is_none());
17812 runner.review_loop().await.unwrap();
17813 runner.gate().await.unwrap();
17814 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17815 assert!(runner.state.gate_allows_progress());
17816 assert_eq!(runner.state.gate_fixes.len(), 1);
17817 assert!(runner.state.merge.is_none());
17818 }
17819 git::git(
17821 &runner.state.repo,
17822 &["commit", "--allow-empty", "-m", "external update"],
17823 )
17824 .await
17825 .unwrap();
17826 runner.merge().await.unwrap();
17827 assert!(runner.state.merge.is_none());
17828 assert_eq!(runner.state.status, RunStatus::Blocked);
17829 }
17830
17831 #[cfg(unix)]
17832 #[tokio::test]
17833 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17834 for waiting_on_gate in [false, true] {
17835 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17836 runner.sync_to_base().await.unwrap();
17837 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17838 runner.state.reconcile_verification_head(&head);
17839 if waiting_on_gate {
17840 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17841 head: head.clone(),
17842 passed: true,
17843 outcomes: Vec::new(),
17844 });
17845 }
17846 let cache = tmp.path().join("cache");
17847 runner.state.config.verify.e2e =
17848 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17849 runner.state.config.verify.gate = vec!["true".into()];
17850 runner.state.config.graph.timeout_resource_wait = 1;
17851 let owner =
17852 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17853 let guard =
17854 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17855 crate::cache::AcquireOutcome::Acquired(g) => g,
17856 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17857 };
17858 runner.review_loop().await.unwrap();
17859 runner.gate().await.unwrap();
17860 assert!(!runner.state.gate_ran);
17861 assert_eq!(runner.state.gate_fixes.len(), 1);
17862 assert!(runner.state.resource_block.is_some());
17863 runner.merge().await.unwrap();
17864 assert!(runner.state.merge.is_none());
17865 drop(guard);
17866 runner.state = RunState::load(&runner.state.id).unwrap();
17867 runner.state.resource_block = None;
17869 runner.review_loop().await.unwrap();
17870 runner.gate().await.unwrap();
17871 assert!(runner.state.gate_allows_progress());
17872 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17873 assert_eq!(runner.state.gate_fixes.len(), 1);
17874 }
17875 }
17876
17877 #[cfg(unix)]
17878 #[tokio::test]
17879 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17880 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17881 runner.sync_to_base().await.unwrap();
17882 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17883 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17884 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17886 runner.review_loop().await.unwrap();
17887 runner.gate().await.unwrap();
17888 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17889 assert_ne!(before, after);
17890 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17891 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17892 assert!(!runner.state.gate_allows_progress());
17893 runner.merge().await.unwrap();
17894 assert!(runner.state.merge.is_none());
17895 assert!(
17896 runner
17897 .state
17898 .verification_history
17899 .iter()
17900 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17901 );
17902
17903 runner.state.reviews[0].clean = false;
17905 runner.state.reviews[0].answered = 0;
17906 runner.state.config.verify.e2e = vec!["true".into()];
17907 runner.review_loop().await.unwrap();
17908 runner.gate().await.unwrap();
17909 runner.merge().await.unwrap();
17910 assert_eq!(runner.state.status, RunStatus::Blocked);
17911 assert!(runner.state.merge.is_none());
17912 assert!(
17913 runner
17914 .state
17915 .verification_blocking_cause()
17916 .unwrap()
17917 .contains("review blocked")
17918 );
17919 }
17920
17921 #[cfg(unix)]
17922 #[tokio::test]
17923 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
17924 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
17925 runner.state.gate_head = None;
17926 runner.state.config.verify.gate =
17927 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
17928 runner.review_loop().await.unwrap();
17929 runner.gate().await.unwrap();
17930 assert_ne!(
17931 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17932 old
17933 );
17934 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
17935 assert!(!runner.state.gate_allows_progress());
17936 assert_eq!(runner.state.status, RunStatus::Blocked);
17937 runner.merge().await.unwrap();
17938 assert!(runner.state.merge.is_none());
17939 }
17940
17941 #[cfg(unix)]
17942 #[tokio::test]
17943 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
17944 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17945 let mut old_review = review_round(true, 0, 1, 1, false, true);
17946 old_review.head = head.clone();
17947 old_review.verified_head = Some(head.clone());
17948 runner.state.reviews = vec![old_review];
17949 runner.state.gate = vec![CommandOutcome {
17950 command: "old gate".into(),
17951 code: None,
17952 output_tail: "timed out after 2400s".into(),
17953 duration_ms: 2_400_000,
17954 resource_blocked: false,
17955 }];
17956 runner.state.gate_ran = true;
17957 runner.state.gate_head = Some(head.clone());
17958 runner.state.land_approval = Some(crate::run::LandApproval {
17959 question: "old-head-approval".into(),
17960 head: head.clone(),
17961 });
17962 runner.state.authorize_base_sync(&task).unwrap();
17963 runner.sync_to_base().await.unwrap();
17964 assert!(!runner.state.base_sync_blocked());
17965 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17966 assert_ne!(head, fresh);
17967 assert_eq!(runner.state.reviews.len(), 1);
17968 assert!(!runner.state.gate_ran);
17969 let saved = &runner.state.base_sync_recovery.authorizations[0];
17970 assert_eq!(saved.reviews[0].head, head);
17971 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
17972 assert!(saved.gate_ran);
17973 assert!(!saved.gate[0].ok());
17974 runner.review_loop().await.unwrap();
17975 runner.gate().await.unwrap();
17976 assert_eq!(runner.state.reviews[0].head, head);
17977 assert_eq!(
17978 runner.state.reviews[0].verified_head.as_deref(),
17979 Some(head.as_str())
17980 );
17981 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
17982 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17983 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
17984 assert!(runner.state.gate_ran);
17985 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17986 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
17987 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
17988 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17989 assert_eq!(
17990 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
17991 "retained change"
17992 );
17993 }
17994
17995 #[cfg(unix)]
17996 #[tokio::test]
17997 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
17998 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17999 let queue = crate::queue::Queue::open();
18000 let mut owner = queue.get(&task).unwrap();
18001 owner.release_operator("test-resume", None);
18002 queue.put(&mut owner).unwrap();
18003 let generation = owner.resumption.as_ref().unwrap().generation;
18004 runner.state.task_instruction = Some(owner.instruction.clone());
18005 runner.state.driver_exited = true;
18006 runner.state.save().unwrap();
18007 let home = crate::run::home();
18008 let _claim = queue.claim(&task).unwrap();
18009 owner.resumption.as_mut().unwrap().at =
18010 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18011 queue.put(&mut owner).unwrap();
18012 assert!(
18013 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18014 .await
18015 .is_err(),
18016 "an approval from before the stop grants nothing"
18017 );
18018 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18019 queue.put(&mut owner).unwrap();
18020 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18021 .await
18022 .unwrap();
18023 assert!(
18024 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18025 .await
18026 .is_err()
18027 );
18028 runner.state = RunState::load(&runner.state.id).unwrap();
18029 assert_eq!(
18030 runner.state.base_sync_recovery.authorizations[0].generation,
18031 Some(generation)
18032 );
18033 runner.sync_to_base().await.unwrap();
18034 let repo = runner.state.repo.clone();
18035 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18036 git::git(&repo, &["checkout", "main"]).await.unwrap();
18037 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18038 git::git(&repo, &["add", "."]).await.unwrap();
18039 git::git(&repo, &["commit", "-m", "move base again"])
18040 .await
18041 .unwrap();
18042 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18043 git::git(&repo, &["checkout", "fix/retained"])
18044 .await
18045 .unwrap();
18046 runner.sync_to_base().await.unwrap();
18047 assert!(!runner.state.base_sync_blocked());
18048 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18049 assert_eq!(runner.state.base_episode_used(), 2);
18050 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18051 assert_ne!(fresh, head);
18052 assert!(!runner.state.base_recovery_pending());
18053 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18055 queue.put(&mut owner).unwrap();
18056 assert!(
18057 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18058 .await
18059 .is_err()
18060 );
18061 runner.state = RunState::load(&runner.state.id).unwrap();
18062 runner.sync_to_base().await.unwrap();
18063 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18064 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18065 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18066 }
18067
18068 #[cfg(unix)]
18069 #[tokio::test]
18070 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18071 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18072 runner.state.base_sync = None;
18073 runner.state.status = RunStatus::Blocked;
18074 let mut second = runner.state.candidates[0].clone();
18075 second.label = 'B';
18076 runner.state.candidates.push(second);
18077 assert!(runner.state.votes.is_empty());
18078 assert!(runner.state.deliberation.is_empty());
18079 runner.deliberate().await.unwrap();
18080 runner.vote().await.unwrap();
18081 assert_eq!(runner.state.status, RunStatus::Blocked);
18082 assert!(runner.state.votes.is_empty());
18083 assert!(runner.state.deliberation.is_empty());
18084 }
18085
18086 fn runner_at(status: RunStatus) -> Runner {
18089 let mut state = RunState::new(
18090 PathBuf::from("/nonexistent/repo"),
18091 "main".to_owned(),
18092 "deadbeef".to_owned(),
18093 "task".to_owned(),
18094 Config::default(),
18095 );
18096 state.config.disk.min_free_bytes = 0;
18098 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18101 state.id = format!(
18102 "20000101-000000-{:04x}",
18103 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18104 );
18105 state.status = status;
18106 Runner {
18107 state,
18108 roles: ResolvedRoles {
18109 implementers: Vec::new(),
18110 judges: Vec::new(),
18111 reviewers: Vec::new(),
18112 fixer: None,
18113 conductor: conductor(),
18114 implementer_roster: Vec::new(),
18115 judge_roster: Vec::new(),
18116 reviewer_roster: Vec::new(),
18117 },
18118 sem: Arc::new(Semaphore::new(1)),
18119 pause: Pause::new(),
18120 interrupt: Pause::new(),
18121 }
18122 }
18123
18124 #[tokio::test]
18125 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18126 crate::run::pin_test_home();
18127 let mut runner = runner_at(RunStatus::Gating);
18128 runner.state.save().unwrap();
18129 let id = runner.state.id.clone();
18130 let passes = runner.state.passes.len();
18131 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18132 let mut claiming = Box::pin(runner.claim_driver());
18133 std::future::poll_fn(|cx| {
18134 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18135 std::task::Poll::Ready(())
18136 })
18137 .await;
18138 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18139 drop(owner);
18140 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18141 .await
18142 .unwrap()
18143 .unwrap();
18144 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18145 assert!(execution.runner.state.resource_block.is_none());
18146 assert_eq!(
18147 execution.runner.state.liveness(false),
18148 crate::run::Liveness::Live
18149 );
18150 drop(execution);
18151 assert!(RunState::load(&id).unwrap().driver_exited);
18152 }
18153
18154 #[tokio::test]
18155 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18156 crate::run::pin_test_home();
18157 let mut runner = runner_at(RunStatus::Gating);
18158 runner.state.save().unwrap();
18159 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18162 assert!(runner.claim_driver().await.is_err());
18163 assert!(runner.state.driver_exited);
18164 assert_eq!(runner.state.driver_pid, None);
18165 let disk = RunState::load(&runner.state.id).unwrap();
18166 assert!(disk.driver_exited);
18167 assert_eq!(disk.driver_pid, None);
18168 assert_eq!(disk.status, RunStatus::Gating);
18169 }
18170
18171 #[tokio::test]
18172 async fn an_early_graph_error_releases_its_driver() {
18173 crate::run::pin_test_home();
18174 let mut runner = runner_at(RunStatus::Blocked);
18175 runner.state.config.graph.selected_implementer = Some("selected".into());
18176 runner.state.save().unwrap();
18177 assert!(runner.execute().await.is_err());
18178 let disk = RunState::load(&runner.state.id).unwrap();
18179 assert!(disk.driver_exited);
18180 assert_eq!(disk.driver_pid, None);
18181 assert_eq!(disk.status, RunStatus::Blocked);
18182 }
18183
18184 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18185 crate::run::MergeRepair {
18186 question: "repair-question".into(),
18187 head: "old-head".into(),
18188 pr: "https://github.com/o/r/pull/42".into(),
18189 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18190 gate: vec![outcome(Some(0), false)],
18191 gate_head: Some("old-head".into()),
18192 gate_fixes: Vec::new(),
18193 pre_gate: Vec::new(),
18194 phase,
18195 failure: None,
18196 }
18197 }
18198
18199 #[tokio::test]
18200 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18201 crate::run::pin_test_home();
18202 let mut runner = runner_at(RunStatus::Landing);
18203 runner.state.merge_repairs.push(merge_repair_request(
18204 crate::run::MergeRepairPhase::Requested,
18205 ));
18206 runner.state.save().unwrap();
18207 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18208 assert!(
18209 runner
18210 .execute()
18211 .await
18212 .unwrap_err()
18213 .to_string()
18214 .contains("already running")
18215 );
18216 assert_eq!(
18217 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18218 crate::run::MergeRepairPhase::Requested
18219 );
18220 drop(claim);
18221 let mut disk = runner.state.clone();
18222 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18223 disk.save().unwrap();
18224 assert!(
18225 runner
18226 .execute()
18227 .await
18228 .unwrap_err()
18229 .to_string()
18230 .contains("decision changed")
18231 );
18232 assert_eq!(
18233 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18234 crate::run::MergeRepairPhase::Complete
18235 );
18236 }
18237
18238 #[test]
18239 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18240 crate::run::pin_test_home();
18241 let mut runner = runner_at(RunStatus::Landing);
18242 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18243 runner.state.reviews = request.reviews.clone();
18244 runner.state.gate = request.gate.clone();
18245 runner.state.gate_ran = true;
18246 runner.state.land_approval = Some(crate::run::LandApproval {
18247 question: "old".into(),
18248 head: "old-head".into(),
18249 });
18250 runner.state.merge_repairs.push(request);
18251 runner.begin_merge_repair_review(0).unwrap();
18252 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18253 assert!(!runner.state.gate_status().ok());
18254 assert!(runner.state.land_approval.is_none());
18255 let disk = RunState::load(&runner.state.id).unwrap();
18256 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18257 assert!(disk.merge_repairs[0].gate[0].ok());
18258 assert_eq!(
18259 disk.merge_repairs[0].phase,
18260 crate::run::MergeRepairPhase::Reviewing
18261 );
18262 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18263 assert!(disk.reviews.is_empty());
18264 }
18265
18266 #[tokio::test]
18267 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18268 crate::run::pin_test_home();
18269 for phase in [
18270 crate::run::MergeRepairPhase::Fixing,
18271 crate::run::MergeRepairPhase::Requested,
18272 ] {
18273 let mut runner = runner_at(RunStatus::Landing);
18274 runner.state.config.graph.land_rounds = 0;
18275 runner.state.merge_repairs.push(merge_repair_request(phase));
18276 runner.state.save().unwrap();
18277 assert!(!runner.repair_merge_review().await.unwrap());
18278 assert_eq!(runner.state.status, RunStatus::Blocked);
18279 assert_eq!(
18280 runner.state.merge_repairs[0].phase,
18281 crate::run::MergeRepairPhase::Failed
18282 );
18283 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18284 assert!(
18285 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18286 "interrupted"
18287 } else {
18288 "budget exhausted"
18289 })
18290 );
18291 assert!(runner.state.merge.is_none());
18292 }
18293 }
18294
18295 #[test]
18299 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18300 crate::run::pin_test_home();
18301 let mut runner = runner_at(RunStatus::Implementing);
18302 let interrupt = Pause::new();
18303 runner.watch_interrupt(interrupt.clone());
18304
18305 interrupt.park_because("task a1b2 asked to run first");
18306
18307 assert!(runner.park_here().expect("park_here"));
18308 assert!(runner.state.parked);
18309 let last = runner.state.events.last().expect("a park event");
18310 assert_eq!(last.node, "park");
18311 assert!(
18312 last.message.contains("task a1b2 asked to run first"),
18313 "expected the interrupt reason in {:?}",
18314 last.message
18315 );
18316 }
18317
18318 #[test]
18321 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18322 crate::run::pin_test_home();
18323 let mut runner = runner_at(RunStatus::Implementing);
18324 runner.state.begin_pass();
18325 let shutdown = Pause::new();
18326 runner.on_pause(shutdown.clone());
18327 let upgrade = ParkCause::Upgrade {
18328 from: "0.121.0".to_owned(),
18329 to: Some("0.121.1".to_owned()),
18330 };
18331 shutdown.park_for(upgrade.clone());
18332 shutdown.park_for(ParkCause::Operator);
18333
18334 assert!(runner.park_here().expect("park_here"));
18335 assert_eq!(
18336 runner.state.passes[0].end,
18337 Some(crate::run::PassEnd::Parked { cause: upgrade })
18338 );
18339 let last = runner.state.events.last().expect("a park event");
18340 assert!(last.message.contains("0.121.0"), "{}", last.message);
18341 }
18342
18343 #[tokio::test]
18344 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18345 crate::run::pin_test_home();
18346 let mut runner = runner_at(RunStatus::Reviewing);
18347 runner.state.save().unwrap();
18348 let mut execution = runner.claim_driver().await.expect("first execution");
18349 assert_eq!(execution.runner.state.passes.len(), 1);
18350 execution
18351 .driver
18352 .finish(&mut execution.runner.state)
18353 .await
18354 .unwrap();
18355 }
18356
18357 #[tokio::test]
18358 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18359 crate::run::pin_test_home();
18360 let mut runner = runner_at(RunStatus::Reviewing);
18361 runner.state.config.agents = vec![conductor()];
18362 runner.state.config.graph.timeout_resource_wait = 0;
18363 runner.state.save().unwrap();
18364 let id = runner.state.id.clone();
18365 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18366 assert!(runner.claim_driver().await.is_err());
18367 drop(owner);
18368 drop(runner);
18369 let mut resumed = Runner::resume(&id).unwrap();
18370 assert_eq!(resumed.state.driver_claimed, Some(false));
18371 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18372 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18373 let mut execution = resumed
18374 .claim_driver()
18375 .await
18376 .expect("retry initial claim after reload");
18377 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18378 assert_eq!(execution.runner.state.passes.len(), 1);
18379 execution
18380 .driver
18381 .finish(&mut execution.runner.state)
18382 .await
18383 .unwrap();
18384 }
18385
18386 #[tokio::test]
18387 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18388 crate::run::pin_test_home();
18389 let mut runner = runner_at(RunStatus::Reviewing);
18390 runner.state.config.agents = vec![conductor()];
18391 runner.state.save().unwrap();
18392 let id = runner.state.id.clone();
18393 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18394 let mut resumed = Runner::resume(&id).unwrap();
18395 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18396 assert!(resumed.claim_driver().await.is_err());
18397 assert!(RunState::load(&id).unwrap().passes.is_empty());
18398 drop(lease);
18399 let mut execution = resumed.claim_driver().await.unwrap();
18400 execution
18401 .driver
18402 .finish(&mut execution.runner.state)
18403 .await
18404 .unwrap();
18405 }
18406
18407 #[tokio::test]
18408 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18409 crate::run::pin_test_home();
18410 let mut runner = runner_at(RunStatus::Reviewing);
18411 let state = &mut runner.state;
18412 state.config.agents = vec![conductor()];
18413 state.driver_claimed = None;
18414 state.driver_pid = Some(std::process::id());
18415 state.driver_started_at = None;
18416 state.begin_pass();
18417 state.save().unwrap();
18418 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18419 std::fs::write(
18420 state.dir().join("driver.json"),
18421 serde_json::to_vec(&serde_json::json!({
18422 "pid": state.driver_pid, "started_at": null,
18423 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18424 }))
18425 .unwrap(),
18426 )
18427 .unwrap();
18428 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18429 drop(lease);
18430 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18431 let mut resumed = Runner::resume(&state.id).unwrap();
18432 assert!(!resumed.state.driver_exited);
18433 let mut execution = resumed
18434 .claim_driver()
18435 .await
18436 .expect("lease proves previous execution ended");
18437 assert_eq!(execution.runner.state.passes.len(), 2);
18438 execution
18439 .driver
18440 .finish(&mut execution.runner.state)
18441 .await
18442 .unwrap();
18443 }
18444
18445 #[tokio::test]
18446 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18447 crate::run::pin_test_home();
18448 let mut runner = runner_at(RunStatus::Reviewing);
18449 let mut json = serde_json::to_value(&runner.state).unwrap();
18450 json.as_object_mut().unwrap().remove("driver_claimed");
18451 runner.state = serde_json::from_value(json).unwrap();
18452 assert_eq!(runner.state.driver_claimed, None);
18453 runner.state.save().unwrap();
18454 let mut execution = runner
18455 .claim_driver()
18456 .await
18457 .expect("the execution lease, not a missing marker, excludes a second driver");
18458 assert_eq!(execution.runner.state.passes.len(), 1);
18459 execution
18460 .driver
18461 .finish(&mut execution.runner.state)
18462 .await
18463 .unwrap();
18464 }
18465
18466 #[tokio::test]
18467 async fn driver_admission_refuses_a_proven_live_driver() {
18468 crate::run::pin_test_home();
18469 let mut runner = runner_at(RunStatus::Reviewing);
18470 let pid = std::process::id();
18471 runner.state.driver_pid = Some(pid);
18472 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18473 assert!(runner.state.driver_started_at.is_some());
18474 runner.state.save().unwrap();
18475 let error = runner
18476 .claim_driver()
18477 .await
18478 .err()
18479 .expect("a live driver must refuse");
18480 assert!(error.to_string().contains("live driver"), "{error:#}");
18481 let saved = RunState::load(&runner.state.id).unwrap();
18482 assert_eq!(saved.driver_pid, Some(pid));
18483 assert_eq!(saved.status, RunStatus::Reviewing);
18484 assert!(saved.passes.is_empty());
18485 }
18486
18487 #[test]
18488 fn parking_resource_halt_persists_the_pass_once() {
18489 crate::run::pin_test_home();
18490 let mut runner = runner_at(RunStatus::Reviewing);
18491 runner.state.begin_pass();
18492 runner.state.resource_block = Some(crate::run::ResourceBlock {
18493 node: "fix".into(),
18494 kind: "shared build cache".into(),
18495 detail: "cancelled".into(),
18496 at: Timestamp::now(),
18497 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18498 });
18499 let cause = ParkCause::Upgrade {
18500 from: "old".into(),
18501 to: Some("new".into()),
18502 };
18503 runner.pause.park_for(cause.clone());
18504 runner.stop_for_resource().unwrap();
18507 assert!(runner.resource_halt().unwrap());
18508 assert!(runner.park_here().unwrap());
18509 let saved = RunState::load(&runner.state.id).unwrap();
18510 assert!(saved.parked);
18511 assert_eq!(saved.status, RunStatus::Reviewing);
18512 assert!(saved.passes[0].ended_at.is_some());
18513 assert_eq!(
18514 saved.passes[0].end,
18515 Some(crate::run::PassEnd::Parked {
18516 cause: cause.clone()
18517 })
18518 );
18519 assert_eq!(
18520 saved.resource_block.unwrap().cause,
18521 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18522 );
18523 assert_eq!(
18524 saved
18525 .events
18526 .iter()
18527 .filter(|event| event.node == "park")
18528 .count(),
18529 1
18530 );
18531 }
18532
18533 #[test]
18541 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18542 crate::run::pin_test_home();
18543 let mut runner = runner_at(RunStatus::Implementing);
18544 let shutdown = Pause::new();
18545 runner.on_pause(shutdown.clone());
18546 let interrupt = Pause::new();
18547 runner.watch_interrupt(interrupt.clone());
18548
18549 assert!(!runner.park_here().expect("park_here"));
18551 assert!(!runner.state.parked);
18552
18553 interrupt.park_because("test");
18555 assert!(!shutdown.parked());
18556 assert!(runner.park_here().expect("park_here"));
18557 }
18558
18559 #[tokio::test]
18573 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18574 crate::run::pin_test_home();
18575 let mut runner = runner_at(RunStatus::Implementing);
18576 let interrupt = Pause::new();
18577 runner.watch_interrupt(interrupt.clone());
18578
18579 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18580 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18581
18582 let node = async move {
18586 started_tx.send(()).expect("send started");
18587 finish_rx.await.expect("recv finish");
18588 "node finished"
18589 };
18590
18591 let interrupter = async move {
18592 started_rx.await.expect("recv started");
18593 interrupt.park_because("higher-priority task waiting");
18595 tokio::task::yield_now().await;
18599 finish_tx.send(()).expect("send finish");
18600 };
18601
18602 let (node_result, ()) = tokio::join!(node, interrupter);
18603 assert_eq!(
18604 node_result, "node finished",
18605 "the in-flight call ran to completion"
18606 );
18607
18608 assert!(runner.park_here().expect("park_here"));
18611 assert!(runner.state.parked);
18612 }
18613
18614 #[test]
18620 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18621 crate::run::pin_test_home();
18622 let mut runner = runner_at(RunStatus::Judging);
18623 runner.state.config.agents = vec![conductor()];
18627 runner.state.candidates = vec![Candidate {
18628 index: 0,
18629 label: 'A',
18630 agent: "alpha".to_owned(),
18631 branch: "magi/x/A".to_owned(),
18632 worktree: PathBuf::from("/nonexistent/worktree"),
18633 summary: "did the thing".to_owned(),
18634 stat: "1 file changed".to_owned(),
18635 files: 1,
18636 commits: 1,
18637 empty: false,
18638 failed: None,
18639 verified_noop: None,
18640 duration_ms: 1234,
18641 folded: false,
18642 }];
18643 let run_id = runner.state.id.clone();
18644
18645 let interrupt = Pause::new();
18646 runner.watch_interrupt(interrupt.clone());
18647 interrupt.park_because("task c3d4 asked to run first");
18648 assert!(runner.park_here().expect("park_here"));
18649
18650 let resumed = Runner::resume(&run_id).expect("resume");
18651 assert_eq!(resumed.state.candidates.len(), 1);
18652 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18653 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18654 assert_eq!(resumed.state.status, runner.state.status);
18655 assert!(
18656 resumed.state.parked,
18657 "still parked until `execute` actually walks the graph again"
18658 );
18659 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18660 }
18661
18662 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18664 let mut q = ask::Question::new(
18665 run.to_owned(),
18666 "implement".to_owned(),
18667 "impl-A".to_owned(),
18668 "Which storage backend should the cache use?".to_owned(),
18669 String::new(),
18670 vec!["SQLite".to_owned(), "Redis".to_owned()],
18671 );
18672 store.put(&mut q).unwrap();
18673 q
18674 }
18675
18676 #[test]
18677 fn a_failed_runs_open_question_is_abandoned() {
18678 ask_test_home();
18679 let store = ask::Questions::open();
18680 let mut runner = runner_at(RunStatus::Failed);
18681 let run = runner.state.id.clone();
18682 let q = ask_open_question(&store, &run);
18683
18684 runner.settle_questions();
18685
18686 let back = store.get(&q.id).unwrap();
18687 assert!(
18688 !back.status.open(),
18689 "the seat that asked died with the run; nobody is left to read an answer"
18690 );
18691 assert!(
18692 back.detail.contains(&run) && back.detail.contains("failed"),
18693 "the reason names what the run became, not just that it is gone: {}",
18694 back.detail
18695 );
18696 }
18697
18698 #[test]
18699 fn a_merged_runs_open_question_is_abandoned_too() {
18700 ask_test_home();
18701 let store = ask::Questions::open();
18702 for status in [RunStatus::Merged, RunStatus::Ready] {
18705 let mut runner = runner_at(status);
18706 let run = runner.state.id.clone();
18707 let q = ask_open_question(&store, &run);
18708
18709 runner.settle_questions();
18710
18711 let back = store.get(&q.id).unwrap();
18712 assert!(
18713 !back.status.open(),
18714 "{status:?} run's question must not outlive the run"
18715 );
18716 }
18717 }
18718
18719 #[test]
18720 fn a_still_resumable_runs_open_question_is_left_alone() {
18721 ask_test_home();
18722 let store = ask::Questions::open();
18723 for status in [RunStatus::Blocked, RunStatus::Stalled] {
18729 let mut runner = runner_at(status);
18730 let run = runner.state.id.clone();
18731 let q = ask_open_question(&store, &run);
18732
18733 runner.settle_questions();
18734
18735 let back = store.get(&q.id).unwrap();
18736 assert!(
18737 back.status.open(),
18738 "{status:?} is still alive; the question must still be waiting"
18739 );
18740 }
18741 }
18742
18743 #[test]
18744 fn settle_questions_never_touches_an_already_answered_question() {
18745 ask_test_home();
18746 let store = ask::Questions::open();
18747 let mut runner = runner_at(RunStatus::Failed);
18748 let run = runner.state.id.clone();
18749 let mut q = ask_open_question(&store, &run);
18750 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18751 .unwrap();
18752 store.put(&mut q).unwrap();
18753
18754 runner.settle_questions();
18759 runner.settle_questions();
18760
18761 let back = store.get(&q.id).unwrap();
18762 assert_eq!(
18763 back.status,
18764 ask::QuestionStatus::Answered,
18765 "a real answer is a decision on record, never overwritten by a sweep"
18766 );
18767 }
18768
18769 #[tokio::test]
18780 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
18781 crate::run::pin_test_home();
18782 let tmp = tempfile::tempdir().expect("tempdir");
18783 let repo = tmp.path().join("repo");
18784 std::fs::create_dir_all(&repo).unwrap();
18785 init_repo(&repo);
18786
18787 let mut config = Config::default();
18788 config.graph.worktree_root = Some(tmp.path().join("wt"));
18789
18790 let mut state = RunState::new(
18791 repo.clone(),
18792 "main".to_owned(),
18793 "deadbeef".to_owned(),
18794 "task".to_owned(),
18795 config,
18796 );
18797 let root = state.worktree_root();
18798 let wt_a = root.join("cand-A");
18799 let wt_b = root.join("cand-B");
18800 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
18801 .await
18802 .expect("worktree A");
18803 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
18804 .await
18805 .expect("worktree B");
18806
18807 state.candidates = vec![
18808 Candidate {
18809 index: 0,
18810 label: 'A',
18811 agent: "alpha".to_owned(),
18812 branch: "magi/x/A".to_owned(),
18813 worktree: wt_a.clone(),
18814 summary: String::new(),
18815 stat: String::new(),
18816 files: 0,
18817 commits: 0,
18818 empty: false,
18819 failed: None,
18820 verified_noop: None,
18821 duration_ms: 0,
18822 folded: false,
18823 },
18824 Candidate {
18825 index: 1,
18826 label: 'B',
18827 agent: "beta".to_owned(),
18828 branch: "magi/x/B".to_owned(),
18829 worktree: wt_b.clone(),
18830 summary: String::new(),
18831 stat: String::new(),
18832 files: 0,
18833 commits: 0,
18834 empty: false,
18835 failed: None,
18836 verified_noop: None,
18837 duration_ms: 0,
18838 folded: false,
18839 },
18840 ];
18841 state.tally = Some(Tally {
18842 first_choice: BTreeMap::from([('A', 1)]),
18843 borda: BTreeMap::new(),
18844 winner: 'A',
18845 rankings: 1,
18846 unanimous_initial: true,
18847 deliberated: false,
18848 changed_votes: 0,
18849 unanimous_final: true,
18850 tie_break: None,
18851 judges: 1,
18852 present: 1,
18853 quorum: 1,
18854 met_quorum: true,
18855 uncontested: None,
18856 });
18857 state.status = RunStatus::Ready;
18858
18859 fold_run(&mut state, false, &crate::run::home())
18860 .await
18861 .expect("fold_run");
18862
18863 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
18864 assert!(
18865 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
18866 "the unmerged winner's branch survives"
18867 );
18868 assert!(
18869 !state.candidates[0].folded,
18870 "the winner is not marked folded"
18871 );
18872
18873 assert!(!wt_b.exists(), "the loser's worktree is removed");
18874 assert!(
18875 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
18876 "the loser's branch is removed"
18877 );
18878 assert!(state.candidates[1].folded, "the loser is marked folded");
18879 }
18880
18881 #[tokio::test]
18882 #[cfg(unix)]
18883 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
18884 use std::os::fd::AsRawFd as _;
18885 use std::time::Duration;
18886
18887 let tmp = tempfile::tempdir().unwrap();
18888 let repo = tmp.path().join("repo");
18889 std::fs::create_dir_all(&repo).unwrap();
18890 init_repo(&repo);
18891 let common = repo.join(".git");
18892 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18893 let home = tmp.path().join("home");
18894 let mut config = Config::default();
18895 config.coordination = crate::config::test_coordination();
18896 config.graph.worktree_root = Some(tmp.path().join("wt"));
18897 config.graph.timeout_resource_wait = 1;
18898 let mut state = RunState::new(
18899 repo.clone(),
18900 "main".into(),
18901 "deadbeef".into(),
18902 "task".into(),
18903 config,
18904 );
18905 let root = state.worktree_root();
18906 let candidate = root.join("cand-A");
18907 let reviewer = root.join("review-1");
18908 let branch = state.branch_for('A');
18909 git::worktree_add_branch(&repo, &candidate, &branch, "main")
18910 .await
18911 .unwrap();
18912 git::worktree_add_detached(&repo, &reviewer, "main")
18913 .await
18914 .unwrap();
18915 git::acquire_worktree_config(&repo).await.unwrap();
18916 state.enabled_worktree_config = true;
18917 state.status = RunStatus::Ready;
18918 state.candidates.push(Candidate {
18919 index: 0,
18920 label: 'A',
18921 agent: "implement".into(),
18922 branch: branch.clone(),
18923 worktree: candidate.clone(),
18924 summary: String::new(),
18925 stat: String::new(),
18926 files: 0,
18927 commits: 0,
18928 empty: false,
18929 failed: None,
18930 verified_noop: None,
18931 duration_ms: 0,
18932 folded: false,
18933 });
18934 state.save_under(&home).unwrap();
18935 let config_before = std::fs::read(common.join("config")).unwrap();
18936 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
18937 .await
18938 .unwrap();
18939 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18942 assert_eq!(
18943 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18944 0
18945 );
18946 let mut fold = Box::pin(fold_run(&mut state, true, &home));
18947 assert!(
18948 tokio::time::timeout(Duration::from_millis(200), &mut fold)
18949 .await
18950 .is_err()
18951 );
18952 assert!(candidate.exists());
18953 assert!(reviewer.exists());
18954 assert!(git::branch_exists(&repo, &branch).await.unwrap());
18955 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
18956 assert_eq!(
18957 git::git(&repo, &["worktree", "list", "--porcelain"])
18958 .await
18959 .unwrap(),
18960 worktrees_before
18961 );
18962 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
18963 .await
18964 .unwrap()
18965 .unwrap_err();
18966 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
18967 drop(fold);
18968 assert!(candidate.exists() && reviewer.exists());
18969 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
18970 assert_eq!(
18971 state.resource_block.as_ref().unwrap().kind,
18972 "repository coordination"
18973 );
18974 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
18975 drop(external);
18976 let mut fold = Box::pin(fold_run(&mut state, true, &home));
18977 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
18978 .await
18979 .unwrap()
18980 .unwrap();
18981 drop(fold);
18982 assert!(removed.contains(&branch));
18983 assert!(!root.exists());
18984 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
18985 assert!(state.candidates[0].folded);
18986 assert!(!state.enabled_worktree_config);
18987 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
18988 let enabled = git::git_raw(
18989 &repo,
18990 &["config", "--local", "--get", "extensions.worktreeConfig"],
18991 )
18992 .await
18993 .unwrap();
18994 assert!(!enabled.ok(), "the last config reference was released");
18995 }
18996
18997 #[tokio::test]
19000 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19001 let tmp = tempfile::tempdir().expect("tempdir");
19002 let repo = tmp.path().join("repo");
19003 std::fs::create_dir_all(&repo).unwrap();
19004 init_repo(&repo);
19005 let home = tmp.path().join("home");
19006
19007 let mut config = Config::default();
19008 config.graph.worktree_root = Some(tmp.path().join("wt"));
19009 let mut state = RunState::new(
19010 repo.clone(),
19011 "main".to_owned(),
19012 "deadbeef".to_owned(),
19013 "task".to_owned(),
19014 config,
19015 );
19016 git::git(&repo, &["branch", "magi/x/A", "main"])
19018 .await
19019 .expect("branch");
19020 state.candidates = vec![Candidate {
19021 index: 0,
19022 label: 'A',
19023 agent: "alpha".to_owned(),
19024 branch: "magi/x/A".to_owned(),
19025 worktree: state.worktree_root().join("cand-A"),
19026 summary: String::new(),
19027 stat: String::new(),
19028 files: 0,
19029 commits: 0,
19030 empty: false,
19031 failed: None,
19032 verified_noop: None,
19033 duration_ms: 0,
19034 folded: true,
19035 }];
19036 state.released_to = Some("20260901-000000-new1".to_owned());
19037 state.released_branches = vec!["magi/x/A".to_owned()];
19038
19039 fold_run(&mut state, true, &home).await.expect("fold_run");
19040 state.status = RunStatus::Merged;
19041 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19042 state.merge = Some(crate::run::MergeOutcome {
19043 mode: MergeMode::Pr,
19044 ok: true,
19045 detail: "confirmed merged".into(),
19046 empty: false,
19047 });
19048 fold_run(&mut state, true, &home)
19049 .await
19050 .expect("merged fold");
19051
19052 assert!(
19053 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19054 "the handed-over branch survives a fold"
19055 );
19056 }
19057
19058 #[tokio::test]
19062 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19063 let tmp = tempfile::tempdir().expect("tempdir");
19064 let repo = tmp.path().join("repo");
19065 std::fs::create_dir_all(&repo).unwrap();
19066 init_repo(&repo);
19067 let run = |args: &[&str]| {
19068 let out = std::process::Command::new("git")
19069 .quiet()
19070 .args(args)
19071 .current_dir(&repo)
19072 .output()
19073 .expect("spawn git");
19074 assert!(out.status.success(), "git {args:?}");
19075 };
19076 run(&["branch", "magi/x/A"]);
19077 run(&["checkout", "-q", "-b", "magi/x/B"]);
19078 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19079 run(&["add", "-A"]);
19080 run(&["commit", "-q", "-m", "work"]);
19081 run(&["checkout", "-q", "main"]);
19082
19083 let probe = RunState::new(
19087 repo.clone(),
19088 "main".to_owned(),
19089 "deadbeef".to_owned(),
19090 "task".to_owned(),
19091 Config::default(),
19092 );
19093 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19094 let bare = tmp.path().join("origin.git");
19095 let out = std::process::Command::new("git")
19096 .quiet()
19097 .args(["init", "-q", "--bare"])
19098 .arg(&bare)
19099 .output()
19100 .expect("spawn git");
19101 assert!(out.status.success(), "git init --bare");
19102 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19103 run(&["push", "-q", "origin", "main"]);
19104
19105 let mut state = RunState::new(
19106 repo.clone(),
19107 "main".to_owned(),
19108 "deadbeef".to_owned(),
19109 "task".to_owned(),
19110 Config::default(),
19111 );
19112 state.seeds = vec![refs::Seed {
19113 token: "magi/27b2/A".to_owned(),
19114 kind: refs::SeedKind::Unresolved,
19115 sha: String::new(),
19116 branch: true,
19117 detail: "no branch or commit named magi/27b2/A".to_owned(),
19118 }];
19119
19120 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19121 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19122 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19123 let detail = empty_candidate_detail(&state, "main");
19124 assert!(detail.starts_with("empty candidate"), "{detail}");
19125 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19126 assert!(detail.contains("--start-from"), "{detail}");
19127 }
19128
19129 #[tokio::test]
19138 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19139 let tmp = tempfile::tempdir().expect("tempdir");
19140 let repo = tmp.path().join("repo");
19141 std::fs::create_dir_all(&repo).unwrap();
19142 init_repo(&repo);
19143
19144 let mut config = Config::default();
19145 config.merge.mode = MergeMode::Local;
19146
19147 let mut state = RunState::new(
19148 repo.clone(),
19149 "main".to_owned(),
19150 "deadbeef".to_owned(),
19151 "task".to_owned(),
19152 config,
19153 );
19154 state.candidates = vec![Candidate {
19155 index: 0,
19156 label: 'A',
19157 agent: "alpha".to_owned(),
19158 branch: "does-not-exist".to_owned(),
19159 worktree: repo.clone(),
19160 summary: String::new(),
19161 stat: String::new(),
19162 files: 0,
19163 commits: 0,
19164 empty: false,
19165 failed: None,
19166 verified_noop: None,
19167 duration_ms: 0,
19168 folded: false,
19169 }];
19170 state.tally = Some(Tally {
19171 first_choice: BTreeMap::from([('A', 1)]),
19172 borda: BTreeMap::new(),
19173 winner: 'A',
19174 rankings: 1,
19175 unanimous_initial: true,
19176 deliberated: false,
19177 changed_votes: 0,
19178 unanimous_final: true,
19179 tie_break: None,
19180 judges: 0,
19181 present: 0,
19182 quorum: 0,
19183 met_quorum: true,
19184 uncontested: Some("only candidate A produced a change".to_owned()),
19185 });
19186 state.reviews = vec![ReviewRound {
19187 round: 1,
19188 head: "deadbeef".to_owned(),
19189 verified_head: None,
19190 verified_at: None,
19191 reviews: Vec::new(),
19192 e2e: Vec::new(),
19193 fix: None,
19194 blocking: 0,
19195 answered: 0,
19196 expected: 0,
19197 clean: true,
19198 verify_retried: false,
19199 e2e_deferred: false,
19200 e2e_defer_reason: None,
19201 progressed: false,
19202 vote_split: false,
19203 reconsideration: Vec::new(),
19204 verdict: None,
19205 }];
19206 state.gate = vec![CommandOutcome {
19207 command: "test".to_owned(),
19208 code: Some(0),
19209 output_tail: String::new(),
19210 duration_ms: 0,
19211 resource_blocked: false,
19212 }];
19213 state.gate_ran = true;
19214 state.status = RunStatus::Ready;
19219 state.merge = Some(MergeOutcome {
19220 mode: MergeMode::Local,
19221 ok: false,
19222 detail: "already concluded".to_owned(),
19223 empty: false,
19224 });
19225
19226 let mut runner = Runner {
19227 state,
19228 roles: ResolvedRoles {
19229 implementers: Vec::new(),
19230 judges: Vec::new(),
19231 reviewers: Vec::new(),
19232 fixer: None,
19233 conductor: conductor(),
19234 implementer_roster: Vec::new(),
19235 judge_roster: Vec::new(),
19236 reviewer_roster: Vec::new(),
19237 },
19238 sem: Arc::new(Semaphore::new(1)),
19239 pause: Pause::new(),
19240 interrupt: Pause::new(),
19241 };
19242
19243 runner.merge().await.expect("merge");
19244
19245 assert_eq!(
19246 runner.state.status,
19247 RunStatus::Ready,
19248 "a concluded run's status must not change on reentry"
19249 );
19250 assert_eq!(
19251 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19252 Some("already concluded"),
19253 "merge must not run again once the node already recorded an outcome"
19254 );
19255 }
19256
19257 #[tokio::test]
19266 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19267 let tmp = tempfile::tempdir().expect("tempdir");
19268 let repo = tmp.path().join("repo");
19269 std::fs::create_dir_all(&repo).unwrap();
19270 init_repo(&repo);
19271
19272 let mut config = Config::default();
19273 config.merge.mode = MergeMode::Local;
19274
19275 let mut state = RunState::new(
19276 repo.clone(),
19277 "main".to_owned(),
19278 "deadbeef".to_owned(),
19279 "task".to_owned(),
19280 config,
19281 );
19282 state.candidates = vec![Candidate {
19283 index: 0,
19284 label: 'A',
19285 agent: "alpha".to_owned(),
19286 branch: "does-not-exist".to_owned(),
19287 worktree: repo.clone(),
19288 summary: String::new(),
19289 stat: String::new(),
19290 files: 0,
19291 commits: 0,
19292 empty: false,
19293 failed: None,
19294 verified_noop: None,
19295 duration_ms: 0,
19296 folded: false,
19297 }];
19298 state.tally = Some(Tally {
19299 first_choice: BTreeMap::from([('A', 1)]),
19300 borda: BTreeMap::new(),
19301 winner: 'A',
19302 rankings: 1,
19303 unanimous_initial: true,
19304 deliberated: false,
19305 changed_votes: 0,
19306 unanimous_final: true,
19307 tie_break: None,
19308 judges: 0,
19309 present: 0,
19310 quorum: 0,
19311 met_quorum: true,
19312 uncontested: Some("only candidate A produced a change".to_owned()),
19313 });
19314 state.reviews = vec![ReviewRound {
19315 round: 1,
19316 head: "deadbeef".to_owned(),
19317 verified_head: None,
19318 verified_at: None,
19319 reviews: Vec::new(),
19320 e2e: Vec::new(),
19321 fix: None,
19322 blocking: 0,
19323 answered: 0,
19324 expected: 0,
19325 clean: true,
19326 verify_retried: false,
19327 e2e_deferred: false,
19328 e2e_defer_reason: None,
19329 progressed: false,
19330 vote_split: false,
19331 reconsideration: Vec::new(),
19332 verdict: None,
19333 }];
19334 state.gate = Vec::new();
19336 state.gate_ran = false;
19337 state.status = RunStatus::Gating;
19338
19339 let mut runner = Runner {
19340 state,
19341 roles: ResolvedRoles {
19342 implementers: Vec::new(),
19343 judges: Vec::new(),
19344 reviewers: Vec::new(),
19345 fixer: None,
19346 conductor: conductor(),
19347 implementer_roster: Vec::new(),
19348 judge_roster: Vec::new(),
19349 reviewer_roster: Vec::new(),
19350 },
19351 sem: Arc::new(Semaphore::new(1)),
19352 pause: Pause::new(),
19353 interrupt: Pause::new(),
19354 };
19355
19356 runner.merge().await.expect("merge");
19357
19358 assert!(
19359 runner.state.merge.is_none(),
19360 "an empty gate must never be read as a passing one: {:?}",
19361 runner.state.merge
19362 );
19363 }
19364
19365 #[tokio::test]
19372 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19373 let tmp = tempfile::tempdir().expect("tempdir");
19374 let repo = tmp.path().join("repo");
19375 std::fs::create_dir_all(&repo).unwrap();
19376 init_repo(&repo);
19377
19378 let config = Config::default();
19380
19381 let mut state = RunState::new(
19382 repo.clone(),
19383 "main".to_owned(),
19384 "deadbeef".to_owned(),
19385 "task".to_owned(),
19386 config,
19387 );
19388 state.candidates = vec![Candidate {
19389 index: 0,
19390 label: 'A',
19391 agent: "alpha".to_owned(),
19392 branch: "does-not-exist".to_owned(),
19393 worktree: repo.clone(),
19394 summary: String::new(),
19395 stat: String::new(),
19396 files: 0,
19397 commits: 0,
19398 empty: false,
19399 failed: None,
19400 verified_noop: None,
19401 duration_ms: 0,
19402 folded: false,
19403 }];
19404 state.tally = Some(Tally {
19405 first_choice: BTreeMap::from([('A', 1)]),
19406 borda: BTreeMap::new(),
19407 winner: 'A',
19408 rankings: 1,
19409 unanimous_initial: true,
19410 deliberated: false,
19411 changed_votes: 0,
19412 unanimous_final: true,
19413 tie_break: None,
19414 judges: 0,
19415 present: 0,
19416 quorum: 0,
19417 met_quorum: true,
19418 uncontested: Some("only candidate A produced a change".to_owned()),
19419 });
19420 state.reviews = vec![ReviewRound {
19421 round: 1,
19422 head: "deadbeef".to_owned(),
19423 verified_head: None,
19424 verified_at: None,
19425 reviews: Vec::new(),
19426 e2e: Vec::new(),
19427 fix: None,
19428 blocking: 0,
19429 answered: 0,
19430 expected: 0,
19431 clean: true,
19432 verify_retried: false,
19433 e2e_deferred: false,
19434 e2e_defer_reason: None,
19435 progressed: false,
19436 vote_split: false,
19437 reconsideration: Vec::new(),
19438 verdict: None,
19439 }];
19440
19441 let mut runner = Runner {
19442 state,
19443 roles: ResolvedRoles {
19444 implementers: Vec::new(),
19445 judges: Vec::new(),
19446 reviewers: Vec::new(),
19447 fixer: None,
19448 conductor: conductor(),
19449 implementer_roster: Vec::new(),
19450 judge_roster: Vec::new(),
19451 reviewer_roster: Vec::new(),
19452 },
19453 sem: Arc::new(Semaphore::new(1)),
19454 pause: Pause::new(),
19455 interrupt: Pause::new(),
19456 };
19457
19458 runner.gate().await.expect("gate");
19459 assert!(
19460 runner.state.gate_ran,
19461 "zero configured commands is still a real attempt, not an unrun gate"
19462 );
19463 assert!(runner.state.gate.is_empty());
19464 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19465 assert_ne!(
19466 runner.state.status,
19467 RunStatus::Blocked,
19468 "a gate with nothing to check must not read as failed"
19469 );
19470
19471 runner.merge().await.expect("merge");
19472 assert_eq!(
19473 runner.state.status,
19474 RunStatus::Ready,
19475 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19476 );
19477 }
19478
19479 #[tokio::test]
19490 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19491 crate::run::pin_test_home();
19492 let home = crate::run::home();
19493
19494 let tmp = tempfile::tempdir().expect("tempdir");
19495 let repo = tmp.path().join("repo");
19496 std::fs::create_dir_all(&repo).unwrap();
19497 init_repo(&repo);
19498 let cache_dir = tmp.path().join("target");
19501
19502 let mut config = Config::default();
19503 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19504 config.graph.timeout_verify = Some(2);
19508 config.graph.timeout_resource_wait = 0;
19509
19510 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19511 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19512 .expect("no io error acquiring directly")
19513 {
19514 crate::cache::AcquireOutcome::Acquired(g) => g,
19515 crate::cache::AcquireOutcome::Busy(b) => {
19516 panic!("expected the direct acquire to win the lease first: {b:?}")
19517 }
19518 };
19519
19520 let mut state = RunState::new(
19521 repo.clone(),
19522 "main".to_owned(),
19523 "deadbeef".to_owned(),
19524 "task".to_owned(),
19525 config,
19526 );
19527 state.candidates = vec![Candidate {
19528 index: 0,
19529 label: 'A',
19530 agent: "alpha".to_owned(),
19531 branch: "does-not-exist".to_owned(),
19532 worktree: repo.clone(),
19533 summary: String::new(),
19534 stat: String::new(),
19535 files: 0,
19536 commits: 0,
19537 empty: false,
19538 failed: None,
19539 verified_noop: None,
19540 duration_ms: 0,
19541 folded: false,
19542 }];
19543 state.tally = Some(Tally {
19544 first_choice: BTreeMap::from([('A', 1)]),
19545 borda: BTreeMap::new(),
19546 winner: 'A',
19547 rankings: 1,
19548 unanimous_initial: true,
19549 deliberated: false,
19550 changed_votes: 0,
19551 unanimous_final: true,
19552 tie_break: None,
19553 judges: 0,
19554 present: 0,
19555 quorum: 0,
19556 met_quorum: true,
19557 uncontested: Some("only candidate A produced a change".to_owned()),
19558 });
19559 state.reviews = vec![ReviewRound {
19560 round: 1,
19561 head: "deadbeef".to_owned(),
19562 verified_head: None,
19563 verified_at: None,
19564 reviews: Vec::new(),
19565 e2e: Vec::new(),
19566 fix: None,
19567 blocking: 0,
19568 answered: 0,
19569 expected: 0,
19570 clean: true,
19571 verify_retried: false,
19572 e2e_deferred: false,
19573 e2e_defer_reason: None,
19574 progressed: false,
19575 vote_split: false,
19576 reconsideration: Vec::new(),
19577 verdict: None,
19578 }];
19579
19580 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19581 state.reviews[0].e2e = vec![CommandOutcome {
19582 command: state.config.verify.e2e[0].clone(),
19583 code: Some(0),
19584 output_tail: String::new(),
19585 duration_ms: 0,
19586 resource_blocked: false,
19587 }];
19588 let mut runner = Runner {
19589 state,
19590 roles: ResolvedRoles {
19591 implementers: Vec::new(),
19592 judges: Vec::new(),
19593 reviewers: Vec::new(),
19594 fixer: None,
19595 conductor: conductor(),
19596 implementer_roster: Vec::new(),
19597 judge_roster: Vec::new(),
19598 reviewer_roster: Vec::new(),
19599 },
19600 sem: Arc::new(Semaphore::new(1)),
19601 pause: Pause::new(),
19602 interrupt: Pause::new(),
19603 };
19604
19605 runner.gate().await.expect("gate");
19606 assert!(runner.state.resource_block.is_none());
19607 assert!(
19608 !runner
19609 .state
19610 .events
19611 .iter()
19612 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19613 "a gate with nothing to run must never request the held cache lease"
19614 );
19615 assert!(
19616 runner.state.gate_ran,
19617 "zero commands is still a real, immediate attempt"
19618 );
19619 assert!(runner.state.gate.is_empty());
19620 assert_ne!(
19621 runner.state.status,
19622 RunStatus::Blocked,
19623 "must not read as resource-blocked on a lease it never asked for"
19624 );
19625 }
19626
19627 #[cfg(unix)]
19630 #[tokio::test]
19631 async fn verification_is_standalone_inside_a_live_worker() {
19632 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19633 if std::env::var_os(CHILD).is_none() {
19634 let home = tempfile::tempdir().unwrap();
19635 let generation = crate::worker::pin_current(home.path()).unwrap();
19636 let output = std::process::Command::new(std::env::current_exe().unwrap())
19637 .args([
19638 "--exact",
19639 "graph::tests::verification_is_standalone_inside_a_live_worker",
19640 "--nocapture",
19641 ])
19642 .env(CHILD, "1")
19643 .env("MAGI_HOME", home.path())
19644 .env("MAGI_WORKER_HOME", home.path())
19645 .env("MAGI_WORKER_ID", "abcdef")
19646 .env("MAGI_WORKER_GENERATION", &generation.id)
19647 .env("MAGI_WORKER_EXE", &generation.executable)
19648 .env("MAGI_RUN", "verification-parent")
19649 .env("MAGI_NODE", "gate")
19650 .env("MAGI_SEAT", "worker-fixture")
19651 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19652 .env("MAGI_NO_AUTOUPDATE", "0")
19653 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19654 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19655 .env(
19656 "VERIFY_EXPECT_TOOLCHAIN",
19657 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19658 )
19659 .env(
19660 "VERIFY_EXPECT_CACHE",
19661 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19662 )
19663 .output()
19664 .unwrap();
19665 assert!(
19666 output.status.success()
19667 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19668 "{}\n{}",
19669 String::from_utf8_lossy(&output.stdout),
19670 String::from_utf8_lossy(&output.stderr)
19671 );
19672 return;
19673 }
19674 let home = crate::run::home();
19675 let directory = home.join("workers/jobs/abcdef");
19676 std::fs::create_dir_all(&directory).unwrap();
19677 let job = crate::worker::Job {
19678 id: "abcdef".into(),
19679 sequence: 1,
19680 kind: "task".into(),
19681 key: "task:verification".into(),
19682 payload: serde_json::json!({}),
19683 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19684 version: Some(env!("CARGO_PKG_VERSION").into()),
19685 status: crate::worker::Status::Running,
19686 result: None,
19687 pid: Some(std::process::id()),
19688 created_at: 0,
19689 finished_at: None,
19690 };
19691 std::fs::write(
19692 directory.join("job.json"),
19693 serde_json::to_vec(&job).unwrap(),
19694 )
19695 .unwrap();
19696 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19697 let assert_worker = || {
19698 assert_eq!(
19699 crate::worker::current_job(&home).unwrap().unwrap().id,
19700 job.id
19701 );
19702 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19703 assert_eq!(
19704 crate::worker::helper_executable(&home).unwrap(),
19705 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19706 );
19707 assert!(
19708 crate::updater::Checker::new(&crate::config::Update {
19709 mode: crate::config::UpdateMode::Notify,
19710 interval: None,
19711 auto_restart: false,
19712 })
19713 .is_none(),
19714 "the driver remains pinned, even while tests are standalone"
19715 );
19716 };
19717 assert_worker();
19718 let repo = home.join("repo");
19719 std::fs::create_dir_all(&repo).unwrap();
19720 init_repo(&repo);
19721 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19724 test -z "${MAGI_WORKER_GENERATION+x}" &&
19725 test -z "${MAGI_WORKER_HOME+x}" &&
19726 test -z "${MAGI_WORKER_EXE+x}" &&
19727 test "$MAGI_HOME/repo" -ef . &&
19728 test "$MAGI_RUN" = verification-parent &&
19729 test "$MAGI_NODE" = gate &&
19730 test "$MAGI_SEAT" = worker-fixture &&
19731 test -n "$MAGI_PROCESS_OWNER" &&
19732 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19733 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19734 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19735 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19736 assert!(
19738 !std::process::Command::new("sh")
19739 .args(["-c", probe])
19740 .current_dir(&repo)
19741 .status()
19742 .unwrap()
19743 .success()
19744 );
19745 let unrelated_home = tempfile::tempdir().unwrap();
19746 assert!(
19747 crate::worker::current_job(unrelated_home.path()).is_err(),
19748 "an inherited worker id cannot be resolved in a unit fixture's home"
19749 );
19750 let mut runner = runner_at(RunStatus::Gating);
19751 runner.state.repo = repo.clone();
19752 runner.state.config.disk.min_free_bytes = 0;
19753 runner.state.config.coordination = crate::config::test_coordination();
19754 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19755 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19756 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19757 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19758 let winner = Candidate {
19759 index: 0,
19760 label: 'A',
19761 agent: "fixture".into(),
19762 branch: "main".into(),
19763 worktree: repo.clone(),
19764 summary: String::new(),
19765 stat: String::new(),
19766 files: 0,
19767 commits: 0,
19768 empty: false,
19769 failed: None,
19770 verified_noop: None,
19771 duration_ms: 0,
19772 folded: false,
19773 };
19774 let (e2e, retried, pids) = run_e2e_with_retry(
19775 &mut runner.state,
19776 &["sh".into(), "-c".into()],
19777 &[format!("{probe} && printf initial")],
19778 &repo,
19779 Duration::from_secs(10),
19780 "initial",
19781 )
19782 .await;
19783 assert!(!retried && pids.is_empty());
19784 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
19785 assert_eq!(e2e[0].output_tail, "initial");
19786 let gate = runner.run_gate(&winner).await.unwrap();
19787 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
19788 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
19791 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
19792 passed: false,
19793 outcomes: Vec::new(),
19794 });
19795 runner.state.save().unwrap();
19796 let mut resumed = RunState::load(&runner.state.id).unwrap();
19797 let wait = crate::repo_lock::WaitPolicy::default();
19798 assert!(matches!(
19799 check_gate_repair_e2e(&mut resumed, &winner, &wait)
19800 .await
19801 .unwrap(),
19802 GateFix::Retry
19803 ));
19804 let saved = RunState::load(&resumed.id).unwrap();
19805 let repair = saved.gate_repair_e2e.unwrap();
19806 assert!(repair.passed);
19807 assert_eq!(repair.outcomes.len(), 1);
19808 assert!(repair.outcomes[0].ok());
19809 let landing = run_landing_gate(&mut resumed, &winner, &wait)
19810 .await
19811 .unwrap();
19812 assert_eq!(landing.len(), 1);
19813 assert!(landing[0].ok(), "{landing:?}");
19814 assert_eq!(
19815 std::fs::read_to_string(repo.join("phases")).unwrap(),
19816 "pre\ngate\ne2e\npre\ngate\n"
19817 );
19818 assert_worker();
19819 }
19820
19821 #[cfg(unix)]
19822 #[tokio::test]
19823 async fn verification_keeps_the_panic_and_the_full_output() {
19824 crate::run::pin_test_home();
19825 let tmp = tempfile::tempdir().unwrap();
19826 let mut config = Config::default();
19827 config.disk.min_free_bytes = 0;
19828 let mut state = RunState::new(
19829 tmp.path().to_owned(),
19830 "main".into(),
19831 "base".into(),
19832 "verification evidence".into(),
19833 config,
19834 );
19835 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
19836 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
19837 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
19838 let mut names = Vec::new();
19839 for _ in 0..2 {
19840 let (outcomes, _) = run_commands(
19841 &mut state,
19842 "gate",
19843 "gate",
19844 0,
19845 &["sh".into(), "-c".into()],
19846 &[script.into()],
19847 tmp.path(),
19848 Duration::from_secs(60),
19849 )
19850 .await;
19851 let tail = &outcomes[0].output_tail;
19852 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
19853 assert!(tail.contains("boom assertion"));
19854 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
19855 let line = tail.lines().next().unwrap();
19856 names.push(line.to_owned());
19857 }
19858 assert_ne!(names[0], names[1], "the second run must not overwrite");
19859 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
19860 assert!(full.len() > 500_000 && full.contains("boom assertion"));
19861 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
19862
19863 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
19865 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
19866 let (outcomes, _) = run_commands(
19867 &mut state,
19868 "gate",
19869 "gate",
19870 0,
19871 &["sh".into(), "-c".into()],
19872 &["echo failure-output; exit 7".into()],
19873 tmp.path(),
19874 Duration::from_secs(10),
19875 )
19876 .await;
19877 assert_eq!(outcomes[0].code, Some(7));
19878 assert!(state.events.iter().any(|e| {
19879 e.node == "gate" && e.message.contains("could not save the full gate output:")
19880 }));
19881 }
19882
19883 #[cfg(unix)]
19884 #[tokio::test]
19885 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
19886 crate::run::pin_test_home();
19887 let tmp = tempfile::tempdir().unwrap();
19888 let mut config = Config::default();
19889 config.disk.min_free_bytes = 0;
19890 let mut state = RunState::new(
19891 tmp.path().to_owned(),
19892 "main".into(),
19893 "base".into(),
19894 "verification evidence".into(),
19895 config,
19896 );
19897 let (outcomes, pids) = run_commands(
19898 &mut state,
19899 "gate",
19900 "gate",
19901 1,
19902 &["sh".into(), "-c".into()],
19903 &[
19904 "printf before-timeout; exec sleep 30".into(),
19905 "kill -TERM $$".into(),
19906 "exit 7".into(),
19907 "printf success".into(),
19908 ],
19909 tmp.path(),
19910 Duration::from_secs(1),
19911 )
19912 .await;
19913 wait_for_timed_out_children_to_die(&pids).await;
19914 assert_eq!(
19915 pids.len(),
19916 1,
19917 "unleased timeout retains its death-check PID"
19918 );
19919 assert_eq!(outcomes.len(), 4);
19920 assert_eq!(outcomes[0].code, None);
19921 assert!(outcomes[0].output_tail.contains("before-timeout"));
19922 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
19923 assert_eq!(outcomes[1].code, None);
19924 assert!(outcomes[1].output_tail.contains("command terminated:"));
19925 assert!(!outcomes[1].output_tail.contains("timed out"));
19926 assert_eq!(outcomes[2].code, Some(7));
19927 assert!(!outcomes[2].output_tail.contains("timed out"));
19928 assert_eq!(outcomes[3].code, Some(0));
19929 assert_eq!(outcomes[3].output_tail, "success");
19930 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
19931 }
19932
19933 #[tokio::test]
19943 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
19944 crate::run::pin_test_home();
19945
19946 let tmp = tempfile::tempdir().expect("tempdir");
19947 let repo = tmp.path().join("repo");
19948 std::fs::create_dir_all(&repo).unwrap();
19949 init_repo(&repo);
19950
19951 let mut config = Config::default();
19952 config.disk.min_free_bytes = 0;
19953 config.verify.gate = vec![
19954 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
19955 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
19956 .to_owned(),
19957 ];
19958
19959 let mut state = RunState::new(
19960 repo.clone(),
19961 "main".to_owned(),
19962 "deadbeef".to_owned(),
19963 "task".to_owned(),
19964 config,
19965 );
19966 let run_id = state.id.clone();
19967 state.candidates = vec![Candidate {
19968 index: 0,
19969 label: 'A',
19970 agent: "alpha".to_owned(),
19971 branch: "does-not-exist".to_owned(),
19972 worktree: repo.clone(),
19973 summary: String::new(),
19974 stat: String::new(),
19975 files: 0,
19976 commits: 0,
19977 empty: false,
19978 failed: None,
19979 verified_noop: None,
19980 duration_ms: 0,
19981 folded: false,
19982 }];
19983 state.tally = Some(Tally {
19984 first_choice: BTreeMap::from([('A', 1)]),
19985 borda: BTreeMap::new(),
19986 winner: 'A',
19987 rankings: 1,
19988 unanimous_initial: true,
19989 deliberated: false,
19990 changed_votes: 0,
19991 unanimous_final: true,
19992 tie_break: None,
19993 judges: 0,
19994 present: 0,
19995 quorum: 0,
19996 met_quorum: true,
19997 uncontested: Some("only candidate A produced a change".to_owned()),
19998 });
19999 state.reviews = vec![ReviewRound {
20000 round: 1,
20001 head: "deadbeef".to_owned(),
20002 verified_head: None,
20003 verified_at: None,
20004 reviews: Vec::new(),
20005 e2e: Vec::new(),
20006 fix: None,
20007 blocking: 0,
20008 answered: 0,
20009 expected: 0,
20010 clean: true,
20011 verify_retried: false,
20012 e2e_deferred: false,
20013 e2e_defer_reason: None,
20014 progressed: false,
20015 vote_split: false,
20016 reconsideration: Vec::new(),
20017 verdict: None,
20018 }];
20019
20020 let mut runner = Runner {
20021 state,
20022 roles: ResolvedRoles {
20023 implementers: Vec::new(),
20024 judges: Vec::new(),
20025 reviewers: Vec::new(),
20026 fixer: None,
20027 conductor: conductor(),
20028 implementer_roster: Vec::new(),
20029 judge_roster: Vec::new(),
20030 reviewer_roster: Vec::new(),
20031 },
20032 sem: Arc::new(Semaphore::new(1)),
20033 pause: Pause::new(),
20034 interrupt: Pause::new(),
20035 };
20036
20037 let started_marker = repo.join("started.marker");
20038 let release_marker = repo.join("release.marker");
20039 let poller = tokio::spawn(async move {
20040 for _ in 0..100 {
20045 if started_marker.exists()
20046 && let Ok(s) = crate::run::RunState::load(&run_id)
20047 && let Some(a) = s.active.get("gate")
20048 {
20049 std::fs::write(&release_marker, b"go").expect("release marker");
20050 return Some(a.clone());
20051 }
20052 tokio::time::sleep(Duration::from_millis(50)).await;
20053 }
20054 None
20055 });
20056
20057 runner.gate().await.expect("gate");
20058 let captured = poller.await.expect("poller task");
20059 let captured = captured.expect(
20060 "the poller never saw a `gate` task entry in run.json while the command was \
20061 still blocked on its own release marker",
20062 );
20063
20064 assert_eq!(captured.task.as_deref(), Some("gate"));
20065 assert_eq!(captured.node, "gate");
20066 assert_eq!(captured.index, Some(1));
20067 assert_eq!(captured.total, Some(1));
20068 assert!(
20069 captured
20070 .command
20071 .as_deref()
20072 .is_some_and(|c| c.contains("started.marker")),
20073 "{captured:?}"
20074 );
20075
20076 assert!(
20077 runner.state.active.is_empty(),
20078 "the entry must be cleared once the command actually finished: {:?}",
20079 runner.state.active
20080 );
20081 assert!(runner.state.gate_ran);
20082 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20083 }
20084
20085 #[tokio::test]
20088 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20089 use crate::verdict::{Finding, ReviewVote, Severity};
20090 crate::run::pin_test_home();
20091 let tmp = tempfile::tempdir().expect("tempdir");
20092 let repo = tmp.path().join("repo");
20093 std::fs::create_dir_all(&repo).unwrap();
20094 init_repo(&repo);
20095
20096 for (severity, vote, expect) in [
20097 (Severity::Major, ReviewVote::Reject, true),
20098 (Severity::Minor, ReviewVote::Reject, false),
20099 (Severity::Major, ReviewVote::Approve, false),
20100 ] {
20101 let mut round = review_round(false, 1, 1, 1, false, true);
20102 round.reviews = vec![ReviewRecord {
20103 reviewer: 1,
20104 agent: "alpha".to_owned(),
20105 summary: String::new(),
20106 findings: vec![Finding {
20107 id: "R1-1-1".to_owned(),
20108 severity,
20109 file: None,
20110 line: None,
20111 title: "t".to_owned(),
20112 detail: String::new(),
20113 }],
20114 vote: Some(vote),
20115 failed: None,
20116 duration_ms: 0,
20117 attempts: 0,
20118 }];
20119 let mut state = RunState::new(
20120 repo.clone(),
20121 "main".to_owned(),
20122 "deadbeef".to_owned(),
20123 "task".to_owned(),
20124 Config::default(),
20125 );
20126 state.reviews = vec![round];
20127 let mut runner = Runner {
20128 state,
20129 roles: ResolvedRoles {
20130 implementers: Vec::new(),
20131 judges: Vec::new(),
20132 reviewers: Vec::new(),
20133 fixer: None,
20134 conductor: conductor(),
20135 implementer_roster: Vec::new(),
20136 judge_roster: Vec::new(),
20137 reviewer_roster: Vec::new(),
20138 },
20139 sem: Arc::new(Semaphore::new(1)),
20140 pause: Pause::new(),
20141 interrupt: Pause::new(),
20142 };
20143 let shell = runner.state.config.shell();
20144 runner
20145 .stop_reviewing("round budget spent", &shell, &repo)
20146 .await
20147 .expect("stop_reviewing");
20148 assert_eq!(runner.state.status, RunStatus::Gating);
20149 assert_eq!(
20150 runner.state.contested_handoff.is_some(),
20151 expect,
20152 "{severity:?} + {vote:?}"
20153 );
20154 }
20155 }
20156
20157 #[tokio::test]
20170 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20171 crate::run::pin_test_home();
20172 let home = crate::run::home();
20173
20174 let tmp = tempfile::tempdir().expect("tempdir");
20175 let repo = tmp.path().join("repo");
20176 std::fs::create_dir_all(&repo).unwrap();
20177 init_repo(&repo);
20178 let head = crate::git::rev_parse(&repo, "HEAD")
20179 .await
20180 .expect("rev-parse");
20181 let cache_dir = tmp.path().join("target");
20184
20185 let mut config = Config::default();
20186 config.verify.e2e = vec![format!(
20187 "CARGO_TARGET_DIR='{}' test -f README.md",
20188 cache_dir.display()
20189 )];
20190 config.graph.review_rounds = 1;
20191 config.graph.timeout_verify = Some(2);
20194 config.graph.timeout_resource_wait = 2;
20195
20196 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20197 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20198 .expect("no io error acquiring directly")
20199 {
20200 crate::cache::AcquireOutcome::Acquired(g) => g,
20201 crate::cache::AcquireOutcome::Busy(b) => {
20202 panic!("expected the direct acquire to win the lease first: {b:?}")
20203 }
20204 };
20205
20206 let mut state = RunState::new(
20207 repo.clone(),
20208 "main".to_owned(),
20209 head.clone(),
20210 "task".to_owned(),
20211 config,
20212 );
20213 state.candidates = vec![Candidate {
20214 index: 0,
20215 label: 'A',
20216 agent: "alpha".to_owned(),
20217 branch: "does-not-exist".to_owned(),
20218 worktree: repo.clone(),
20219 summary: String::new(),
20220 stat: String::new(),
20221 files: 0,
20222 commits: 0,
20223 empty: false,
20224 failed: None,
20225 verified_noop: None,
20226 duration_ms: 0,
20227 folded: false,
20228 }];
20229 state.tally = Some(Tally {
20230 first_choice: BTreeMap::from([('A', 1)]),
20231 borda: BTreeMap::new(),
20232 winner: 'A',
20233 rankings: 1,
20234 unanimous_initial: true,
20235 deliberated: false,
20236 changed_votes: 0,
20237 unanimous_final: true,
20238 tie_break: None,
20239 judges: 0,
20240 present: 0,
20241 quorum: 0,
20242 met_quorum: true,
20243 uncontested: Some("only candidate A produced a change".to_owned()),
20244 });
20245 state.reviews = vec![ReviewRound {
20249 round: 1,
20250 head: head.clone(),
20251 verified_head: None,
20252 verified_at: None,
20253 reviews: Vec::new(),
20254 e2e: Vec::new(),
20255 fix: None,
20256 blocking: 1,
20257 answered: 1,
20258 expected: 1,
20259 clean: false,
20260 verify_retried: false,
20261 e2e_deferred: true,
20262 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20263 progressed: false,
20264 vote_split: false,
20265 reconsideration: Vec::new(),
20266 verdict: None,
20267 }];
20268
20269 let mut runner = Runner {
20270 state,
20271 roles: ResolvedRoles {
20272 implementers: Vec::new(),
20273 judges: Vec::new(),
20274 reviewers: Vec::new(),
20275 fixer: None,
20276 conductor: conductor(),
20277 implementer_roster: Vec::new(),
20278 judge_roster: Vec::new(),
20279 reviewer_roster: Vec::new(),
20280 },
20281 sem: Arc::new(Semaphore::new(1)),
20282 pause: Pause::new(),
20283 interrupt: Pause::new(),
20284 };
20285
20286 let shell = runner.state.config.shell();
20287 runner
20288 .stop_reviewing("round budget spent", &shell, &repo)
20289 .await
20290 .expect("stop_reviewing");
20291
20292 let last = runner.state.reviews.last().expect("round record");
20293 assert_eq!(
20294 last.e2e_status(),
20295 E2eStatus::ResourceBlocked,
20296 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20297 failed: {last:?}"
20298 );
20299 assert_eq!(
20300 last.verified_head.as_deref(),
20301 Some(head.as_str()),
20302 "which commit this attempt targeted is known even though nothing finished checking \
20303 it"
20304 );
20305 let first_attempt_at = last
20306 .verified_at
20307 .expect("when this attempt ran is known too");
20308 assert_ne!(
20309 runner.state.status,
20310 RunStatus::Blocked,
20311 "contention is evidence about the machine, not the patch — it must not settle the \
20312 run as blocked: {:?}",
20313 runner.state.status
20314 );
20315 assert!(
20316 !runner
20317 .state
20318 .events
20319 .iter()
20320 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20321 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20322 runner.state.events
20323 );
20324
20325 runner
20330 .stop_reviewing("round budget spent", &shell, &repo)
20331 .await
20332 .expect("stop_reviewing retry");
20333 assert_eq!(
20334 runner.state.reviews.len(),
20335 1,
20336 "no new round was started: {:?}",
20337 runner.state.reviews
20338 );
20339 let last = runner.state.reviews.last().expect("round record");
20340 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20341 assert!(
20342 last.verified_at.expect("still known") > first_attempt_at,
20343 "a second reentry must be a fresh attempt, not a stale copy of the first"
20344 );
20345 assert_ne!(runner.state.status, RunStatus::Blocked);
20346
20347 held.release();
20348 }
20349
20350 #[cfg(unix)]
20351 #[tokio::test]
20352 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20353 ask_test_home();
20354 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20355 let tmp = tempfile::tempdir().unwrap();
20356 let repo = tmp.path().join("repo");
20357 std::fs::create_dir(&repo).unwrap();
20358 init_repo(&repo);
20359 let run_git = |args: &[&str]| {
20360 let out = std::process::Command::new("git")
20361 .args(args)
20362 .current_dir(&repo)
20363 .output()
20364 .unwrap();
20365 assert!(
20366 out.status.success(),
20367 "{args:?}: {}",
20368 String::from_utf8_lossy(&out.stderr)
20369 );
20370 };
20371 let origin = tmp.path().join("origin.git");
20372 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20373 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20374 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20375 run_git(&["add", "spec.rs"]);
20376 run_git(&["commit", "-m", "initial spec"]);
20377 run_git(&["push", "origin", "main"]);
20378 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20379 run_git(&["checkout", "-b", "fix/review"]);
20380 std::fs::write(
20381 repo.join("check.rs"),
20382 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20383 )
20384 .unwrap();
20385 run_git(&["add", "check.rs"]);
20386 run_git(&["commit", "-m", "use spec"]);
20387 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20388 run_git(&["checkout", "main"]);
20389 std::fs::write(
20390 repo.join("spec.rs"),
20391 "struct AgentSpec { max_turn_secs: u64 }\n",
20392 )
20393 .unwrap();
20394 run_git(&["add", "spec.rs"]);
20395 run_git(&["commit", "-m", "extend spec"]);
20396 run_git(&["push", "origin", "main"]);
20397 run_git(&["checkout", "fix/review"]);
20398
20399 let mut runner = runner_at(RunStatus::Reviewing);
20400 runner.state.repo = repo.clone();
20401 runner.state.base_commit = base;
20402 runner.state.pr_head = Some("fix/review".into());
20403 runner.state.config.graph.review_rounds = max_rounds;
20404 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20405 runner.state.config.graph.sessions = false;
20406 runner.state.config.disk.min_free_bytes = 0;
20407 runner.state.config.verify.e2e =
20408 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20409 runner.state.candidates = vec![Candidate {
20410 index: 0,
20411 label: 'A',
20412 agent: "fixture".into(),
20413 branch: "fix/review".into(),
20414 worktree: repo.clone(),
20415 summary: String::new(),
20416 stat: String::new(),
20417 files: 1,
20418 commits: 1,
20419 empty: false,
20420 failed: None,
20421 verified_noop: None,
20422 duration_ms: 0,
20423 folded: false,
20424 }];
20425 runner.state.tally = Some(Tally {
20426 first_choice: BTreeMap::new(),
20427 borda: BTreeMap::new(),
20428 winner: 'A',
20429 rankings: 0,
20430 unanimous_initial: true,
20431 deliberated: false,
20432 changed_votes: 0,
20433 unanimous_final: true,
20434 tie_break: None,
20435 judges: 0,
20436 present: 0,
20437 quorum: 0,
20438 met_quorum: true,
20439 uncontested: Some("review-only".into()),
20440 });
20441 let mut last = review_round(false, 0, 1, 1, false, false);
20442 last.head = reviewed.clone();
20443 last.e2e_deferred = true;
20444 last.e2e[0].resource_blocked = true;
20445 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20446 if max_rounds == 1 {
20447 last.e2e.clear();
20448 }
20449 runner.state.reviews = vec![last];
20450 let calls = tmp.path().join("fix-calls");
20451 let mut fixer = spec("repair");
20452 fixer
20453 .env
20454 .insert("FIX_CALLS".into(), calls.display().to_string());
20455 fixer.command = vec!["sh".into(), "-c".into(), r#"
20456 echo fix >> "$FIX_CALLS"
20457 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20458 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20459 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20460 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20461 "#.into()];
20462 let mut reviewer = spec("reviewer");
20463 reviewer.command = vec![
20464 "sh".into(),
20465 "-c".into(),
20466 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20467 .into(),
20468 ];
20469 runner.roles.fixer = Some(vec![fixer]);
20470 runner.roles.reviewers = vec![reviewer];
20471
20472 runner.sync_to_base().await.unwrap();
20473 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20474 assert_ne!(rebased, reviewed);
20475 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20476 if reentry {
20477 let shell = runner.state.config.shell();
20480 runner
20481 .stop_unresolved_review("deferred verification", &shell, &repo)
20482 .await
20483 .unwrap();
20484 assert!(runner.catchup_repairable());
20485 assert!(!calls.exists());
20486 runner.state = RunState::load(&runner.state.id).unwrap();
20487 assert!(runner.catchup_repairable());
20488 }
20489 runner.review_loop().await.unwrap();
20490 let first = &runner.state.reviews[0];
20491 assert_eq!(first.head, reviewed);
20492 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20493 assert!(
20494 first.verify_retried,
20495 "a reproduced build failure gets exactly one retry: {:?}",
20496 first.e2e
20497 );
20498 assert!(first.e2e[0].output_tail.contains("E0063"));
20499 assert!(!runner.catchup_repairable());
20500 if max_rounds == 2 {
20501 assert_eq!(
20502 runner.state.status,
20503 RunStatus::Gating,
20504 "{}",
20505 crate::report::run(&runner.state)
20506 );
20507 assert_eq!(runner.state.reviews.len(), 2);
20508 assert!(first.fix.as_ref().unwrap().committed);
20509 assert!(runner.state.reviews[1].clean);
20510 assert_eq!(
20511 runner
20512 .state
20513 .reviews
20514 .iter()
20515 .filter(|r| r.fix.is_some())
20516 .count(),
20517 1
20518 );
20519 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20520 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20521 } else {
20522 assert_eq!(runner.state.status, RunStatus::Blocked);
20523 assert!(first.fix.is_none());
20524 assert!(!calls.exists());
20525 let recovery = crate::recovery::assess(
20526 "prior",
20527 Some(&runner.state),
20528 &runner.landing_base(),
20529 crate::recovery::Relation::Diverged,
20530 );
20531 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20532 let reasons = recovery.reasons.join("\n");
20533 assert!(
20534 reasons.contains(&reviewed) && reasons.contains(&rebased),
20535 "{reasons}"
20536 );
20537 assert!(
20538 reasons.contains("base sync to")
20539 && reasons.contains("additional review is required"),
20540 "{reasons}"
20541 );
20542 }
20543 }
20544 }
20545
20546 #[tokio::test]
20558 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20559 crate::run::pin_test_home();
20560 let home = crate::run::home();
20561
20562 let tmp = tempfile::tempdir().expect("tempdir");
20563 let repo = tmp.path().join("repo");
20564 std::fs::create_dir_all(&repo).unwrap();
20565 init_repo(&repo);
20566 let head = crate::git::rev_parse(&repo, "HEAD")
20567 .await
20568 .expect("rev-parse");
20569 let cache_dir = tmp.path().join("target");
20570
20571 let mut config = Config::default();
20572 config.verify.e2e = vec![format!(
20573 "CARGO_TARGET_DIR='{}' test -f README.md",
20574 cache_dir.display()
20575 )];
20576 config.graph.review_rounds = 1;
20577 config.graph.timeout_verify = Some(2);
20578 config.graph.timeout_resource_wait = 2;
20579
20580 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20581 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20582 .expect("no io error acquiring directly")
20583 {
20584 crate::cache::AcquireOutcome::Acquired(g) => g,
20585 crate::cache::AcquireOutcome::Busy(b) => {
20586 panic!("expected the direct acquire to win the lease first: {b:?}")
20587 }
20588 };
20589
20590 let mut state = RunState::new(
20591 repo.clone(),
20592 "main".to_owned(),
20593 head.clone(),
20594 "task".to_owned(),
20595 config,
20596 );
20597 state.candidates = vec![Candidate {
20598 index: 0,
20599 label: 'A',
20600 agent: "alpha".to_owned(),
20601 branch: "does-not-exist".to_owned(),
20602 worktree: repo.clone(),
20603 summary: String::new(),
20604 stat: String::new(),
20605 files: 0,
20606 commits: 0,
20607 empty: false,
20608 failed: None,
20609 verified_noop: None,
20610 duration_ms: 0,
20611 folded: false,
20612 }];
20613 state.tally = Some(Tally {
20614 first_choice: BTreeMap::from([('A', 1)]),
20615 borda: BTreeMap::new(),
20616 winner: 'A',
20617 rankings: 1,
20618 unanimous_initial: true,
20619 deliberated: false,
20620 changed_votes: 0,
20621 unanimous_final: true,
20622 tie_break: None,
20623 judges: 0,
20624 present: 0,
20625 quorum: 0,
20626 met_quorum: true,
20627 uncontested: Some("only candidate A produced a change".to_owned()),
20628 });
20629 state.reviews = vec![ReviewRound {
20633 round: 1,
20634 head: head.clone(),
20635 verified_head: Some(head.clone()),
20636 verified_at: Some(jiff::Timestamp::now()),
20637 reviews: Vec::new(),
20638 e2e: vec![CommandOutcome {
20639 command: format!(
20640 "CARGO_TARGET_DIR='{}' test -f README.md",
20641 cache_dir.display()
20642 ),
20643 code: None,
20644 output_tail: "waiting for the shared build cache".to_owned(),
20645 duration_ms: 0,
20646 resource_blocked: true,
20647 }],
20648 fix: None,
20649 blocking: 1,
20650 answered: 1,
20651 expected: 1,
20652 clean: false,
20653 verify_retried: false,
20654 e2e_deferred: false,
20655 e2e_defer_reason: None,
20656 progressed: false,
20657 vote_split: false,
20658 reconsideration: Vec::new(),
20659 verdict: None,
20660 }];
20661
20662 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20663 let mut runner = Runner {
20664 state,
20665 roles: ResolvedRoles {
20666 implementers: Vec::new(),
20667 judges: Vec::new(),
20668 reviewers: Vec::new(),
20669 fixer: None,
20670 conductor: conductor(),
20671 implementer_roster: Vec::new(),
20672 judge_roster: Vec::new(),
20673 reviewer_roster: Vec::new(),
20674 },
20675 sem: Arc::new(Semaphore::new(1)),
20676 pause: Pause::new(),
20677 interrupt: Pause::new(),
20678 };
20679
20680 runner.review_loop().await.expect("review_loop");
20685
20686 assert_eq!(
20687 runner.state.reviews.len(),
20688 1,
20689 "no new round was started on top of the unresolved one: {:?}",
20690 runner.state.reviews
20691 );
20692 let last = &runner.state.reviews[0];
20693 assert_eq!(
20694 last.e2e_status(),
20695 E2eStatus::ResourceBlocked,
20696 "still contended: {last:?}"
20697 );
20698 assert!(
20699 last.verified_at.expect("still known") > first_attempt_at,
20700 "review_loop must have actually retried the check, not left it exactly as found"
20701 );
20702 assert_ne!(
20703 runner.state.status,
20704 RunStatus::Blocked,
20705 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20706 runner.state.status
20707 );
20708
20709 held.release();
20710 }
20711
20712 #[tokio::test]
20713 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20714 crate::run::pin_test_home();
20715 for status in [RunStatus::Blocked, RunStatus::Gating] {
20716 let mut runner = runner_at(status);
20717 runner.state.config.graph.land = true;
20718 runner.state.gate_ran = true;
20719 runner.state.merge = Some(MergeOutcome {
20720 mode: MergeMode::Pr,
20721 ok: false,
20722 detail: "the fixer produced no commit".into(),
20723 empty: false,
20724 });
20725 let url = "https://example.invalid/o/r/pull/1";
20726 runner.state.pr = Some(crate::run::PrRecord {
20727 url: url.into(),
20728 number: 1,
20729 state: "open".into(),
20730 checks: "red".into(),
20731 round: 1,
20732 rounds: 1,
20733 red_at_merge: Vec::new(),
20734 });
20735 runner.state.save().unwrap();
20736 runner.execute_graph().await.unwrap();
20740 assert_eq!(runner.state.status, RunStatus::Blocked);
20741 assert!(runner.state.land_sync_pending);
20742 assert!(runner.state.events.iter().any(|event|
20743 event.node == "land" && event.message == format!("watching {url}")));
20744 assert_eq!(
20745 runner.state.merge.as_ref().unwrap().detail,
20746 "the fixer produced no commit"
20747 );
20748 }
20749 }
20750
20751 #[tokio::test]
20752 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20753 crate::run::pin_test_home();
20754 let tmp = tempfile::tempdir().expect("tempdir");
20755 let repo = tmp.path().join("repo");
20756 std::fs::create_dir_all(&repo).unwrap();
20757 init_repo(&repo);
20758
20759 let mut config = Config::default();
20760 config.merge.mode = MergeMode::Pr;
20761 config.graph.land = true;
20762 config.graph.land_approval = false;
20763
20764 let mut state = RunState::new(
20765 repo.clone(),
20766 "main".to_owned(),
20767 "deadbeef".to_owned(),
20768 "task".to_owned(),
20769 config,
20770 );
20771 state.candidates = vec![Candidate {
20772 index: 0,
20773 label: 'A',
20774 agent: "alpha".to_owned(),
20775 branch: "does-not-exist".to_owned(),
20776 worktree: repo.clone(),
20777 summary: String::new(),
20778 stat: String::new(),
20779 files: 0,
20780 commits: 0,
20781 empty: false,
20782 failed: None,
20783 verified_noop: None,
20784 duration_ms: 0,
20785 folded: false,
20786 }];
20787 state.tally = Some(Tally {
20788 first_choice: BTreeMap::from([('A', 1)]),
20789 borda: BTreeMap::new(),
20790 winner: 'A',
20791 rankings: 1,
20792 unanimous_initial: true,
20793 deliberated: false,
20794 changed_votes: 0,
20795 unanimous_final: true,
20796 tie_break: None,
20797 judges: 0,
20798 present: 0,
20799 quorum: 0,
20800 met_quorum: true,
20801 uncontested: Some("only candidate A produced a change".to_owned()),
20802 });
20803 state.reviews = vec![ReviewRound {
20804 round: 1,
20805 head: "deadbeef".to_owned(),
20806 verified_head: None,
20807 verified_at: None,
20808 reviews: Vec::new(),
20809 e2e: Vec::new(),
20810 fix: None,
20811 blocking: 0,
20812 answered: 0,
20813 expected: 0,
20814 clean: true,
20815 verify_retried: false,
20816 e2e_deferred: false,
20817 e2e_defer_reason: None,
20818 progressed: false,
20819 vote_split: false,
20820 reconsideration: Vec::new(),
20821 verdict: None,
20822 }];
20823 state.gate = vec![CommandOutcome {
20824 command: "test".to_owned(),
20825 code: Some(0),
20826 output_tail: String::new(),
20827 duration_ms: 0,
20828 resource_blocked: false,
20829 }];
20830 state.gate_ran = true;
20831 state.status = RunStatus::Landing;
20835 state.merge = Some(MergeOutcome {
20836 mode: MergeMode::Pr,
20837 ok: true,
20838 detail: "https://example.invalid/x/y/pull/1".to_owned(),
20839 empty: false,
20840 });
20841
20842 ask_test_home();
20846 state.save().expect("save landing run");
20849 let store = ask::Questions::open();
20850 let q = ask_open_question(&store, &state.id);
20851
20852 let mut runner = Runner {
20853 state,
20854 roles: ResolvedRoles {
20855 implementers: Vec::new(),
20856 judges: Vec::new(),
20857 reviewers: Vec::new(),
20858 fixer: None,
20859 conductor: conductor(),
20860 implementer_roster: Vec::new(),
20861 judge_roster: Vec::new(),
20862 reviewer_roster: Vec::new(),
20863 },
20864 sem: Arc::new(Semaphore::new(1)),
20865 pause: Pause::new(),
20866 interrupt: Pause::new(),
20867 };
20868
20869 runner.execute().await.expect("execute");
20874
20875 assert_eq!(
20876 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20877 Some("https://example.invalid/x/y/pull/1"),
20878 "reentry must not push again or open a second pull request over the \
20879 one `land` is already watching"
20880 );
20881 assert_ne!(
20882 runner.state.status,
20883 RunStatus::Landing,
20884 "land could not actually reach the fake pull request, so it must \
20885 have given up rather than left the run silently parked forever"
20886 );
20887 assert_eq!(runner.state.status, RunStatus::Blocked);
20891 assert!(
20892 store.get(&q.id).unwrap().status.open(),
20893 "Blocked is still alive; settle_questions must have been a no-op here"
20894 );
20895 }
20896
20897 fn state_with_round(round: ReviewRound) -> RunState {
20898 let mut s = RunState::new(
20899 PathBuf::from("/repo"),
20900 "main".to_owned(),
20901 "abc1234".to_owned(),
20902 "add retries".to_owned(),
20903 Config::default(),
20904 );
20905 s.reviews = vec![round];
20906 s
20907 }
20908
20909 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
20910 crate::verdict::Finding {
20911 id: id.to_owned(),
20912 severity,
20913 file: None,
20914 line: None,
20915 title: title.to_owned(),
20916 detail: String::new(),
20917 }
20918 }
20919
20920 #[test]
20921 fn pr_body_names_open_findings_and_declined_ones() {
20922 let round = ReviewRound {
20923 round: 2,
20924 head: "deadbee".to_owned(),
20925 verified_head: None,
20926 verified_at: None,
20927 reviews: vec![ReviewRecord {
20928 attempts: 0,
20929 reviewer: 1,
20930 agent: "alpha".to_owned(),
20931 summary: String::new(),
20932 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
20933 vote: None,
20934 failed: None,
20935 duration_ms: 0,
20936 }],
20937 e2e: vec![CommandOutcome {
20938 command: "cargo test".to_owned(),
20939 code: Some(0),
20940 output_tail: String::new(),
20941 duration_ms: 0,
20942 resource_blocked: false,
20943 }],
20944 verify_retried: false,
20945 e2e_deferred: false,
20946 e2e_defer_reason: None,
20947 fix: Some(FixRecord {
20948 agent: "alpha".to_owned(),
20949 addressed: Vec::new(),
20950 rejected: vec![crate::verdict::Rejection {
20951 id: "R1-1-1".to_owned(),
20952 why: "not reachable from any caller".to_owned(),
20953 }],
20954 notes: String::new(),
20955 committed: true,
20956 failed: None,
20957 duration_ms: 0,
20958 continuation: None,
20959 }),
20960 blocking: 0,
20961 answered: 1,
20962 expected: 1,
20963 clean: false,
20964 progressed: true,
20965 vote_split: false,
20966 reconsideration: Vec::new(),
20967 verdict: None,
20968 };
20969 let state = state_with_round(round);
20970 let body = pr_message(&state, 'A').body;
20971
20972 assert!(body.contains("add retries"), "the task must still be there");
20973 assert!(body.contains("R2-1-1"), "{body}");
20974 assert!(!body.contains("unused import"), "{body}");
20975 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
20976 assert!(
20977 !body.contains("not reachable from any caller"),
20978 "the reason it was declined: {body}"
20979 );
20980 }
20981
20982 #[test]
20983 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
20984 let round = ReviewRound {
20985 round: 1,
20986 head: "deadbee".to_owned(),
20987 verified_head: None,
20988 verified_at: None,
20989 reviews: vec![ReviewRecord {
20990 attempts: 0,
20991 reviewer: 1,
20992 agent: "alpha".to_owned(),
20993 summary: String::new(),
20994 findings: Vec::new(),
20995 vote: None,
20996 failed: None,
20997 duration_ms: 0,
20998 }],
20999 e2e: Vec::new(),
21000 verify_retried: false,
21001 e2e_deferred: false,
21002 e2e_defer_reason: None,
21003 fix: None,
21004 blocking: 0,
21005 answered: 1,
21006 expected: 1,
21007 clean: true,
21008 progressed: false,
21009 vote_split: false,
21010 reconsideration: Vec::new(),
21011 verdict: None,
21012 };
21013 let state = state_with_round(round);
21014 let body = pr_message(&state, 'A').body;
21015 assert!(!body.contains("Open review findings"), "{body}");
21016 assert!(!body.contains("Declined"), "{body}");
21017 }
21018
21019 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21020 let mut state = RunState::new(
21021 PathBuf::from("/repo"),
21022 "main".to_owned(),
21023 "abc1234".to_owned(),
21024 instruction.to_owned(),
21025 Config::default(),
21026 );
21027 state.config.disk.min_free_bytes = 0;
21029 state.candidates.push(Candidate {
21030 index: 0,
21031 label: 'A',
21032 agent: "alpha".to_owned(),
21033 branch: "magi/x/A".to_owned(),
21034 worktree: PathBuf::from("/wt"),
21035 summary: summary.to_owned(),
21036 stat: String::new(),
21037 files: 1,
21038 commits: 1,
21039 empty: false,
21040 failed: None,
21041 verified_noop: None,
21042 folded: false,
21043 duration_ms: 0,
21044 });
21045 state
21046 }
21047
21048 fn review_state(subjects: &[&str]) -> RunState {
21049 let mut state = state_with_summary(
21050 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21051 "",
21052 );
21053 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21054 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21055 state
21056 }
21057
21058 #[tokio::test]
21061 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21062 ask_test_home();
21063 let tmp = tempfile::tempdir().unwrap();
21064 let repo = tmp.path().join("repo");
21065 std::fs::create_dir_all(&repo).unwrap();
21066 init_repo(&repo);
21067 let origin = tmp.path().join("origin.git");
21068 let g = |dir: &Path, args: &[&str]| {
21069 let out = std::process::Command::new("git")
21070 .args(args)
21071 .current_dir(dir)
21072 .quiet()
21073 .output()
21074 .expect("spawn git");
21075 assert!(
21076 out.status.success(),
21077 "git {args:?}: {}",
21078 String::from_utf8_lossy(&out.stderr)
21079 );
21080 };
21081 g(
21082 tmp.path(),
21083 &[
21084 "clone",
21085 "--bare",
21086 "-q",
21087 repo.to_str().unwrap(),
21088 origin.to_str().unwrap(),
21089 ],
21090 );
21091 g(
21092 &repo,
21093 &["remote", "add", "origin", origin.to_str().unwrap()],
21094 );
21095 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21096
21097 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21099 g(&repo, &["add", "-A"]);
21100 g(
21101 &repo,
21102 &["commit", "-q", "-m", "chore(deps): update a crate"],
21103 );
21104 g(&repo, &["push", "-q", "origin", "main"]);
21105 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21106 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21107 g(&repo, &["add", "-A"]);
21108 g(
21109 &repo,
21110 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21111 );
21112 g(&repo, &["checkout", "-q", "main"]);
21113
21114 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21115 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21116 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21118 assert_eq!(fallback, c1);
21119
21120 let mut state = review_state(&[
21121 "chore(deps): update a crate",
21122 "fix(daemon): apply a chosen action",
21123 ]);
21124 state.repo = repo.clone();
21125 state.base_branch = "main".to_owned();
21126 state.base_commit = c1;
21127 refresh_reviewed_commits(&mut state, "feat/own").await;
21128 assert_eq!(
21129 state.reviewed_commits,
21130 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21131 );
21132 assert_eq!(
21133 review_title(&state).as_deref(),
21134 Some("fix(daemon): apply a chosen action")
21135 );
21136 assert_eq!(
21137 leaked_subjects(&state, "feat/own").await,
21138 Some(vec!["chore(deps): update a crate".to_owned()])
21139 );
21140 state.config.merge.remote = "nowhere".to_owned();
21143 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21144 }
21145
21146 struct SummaryFixture {
21150 _tmp: tempfile::TempDir,
21151 repo: PathBuf,
21152 reply: PathBuf,
21153 runner: Runner,
21154 }
21155
21156 fn summary_fixture() -> SummaryFixture {
21157 ask_test_home();
21158 let tmp = tempfile::tempdir().unwrap();
21159 let repo = tmp.path().join("repo");
21160 std::fs::create_dir_all(&repo).unwrap();
21161 init_repo(&repo);
21162 let origin = tmp.path().join("origin.git");
21163 let g = |args: &[&str]| {
21164 let out = std::process::Command::new("git")
21165 .args(args)
21166 .current_dir(&repo)
21167 .quiet()
21168 .output()
21169 .expect("spawn git");
21170 assert!(
21171 out.status.success(),
21172 "git {args:?}: {}",
21173 String::from_utf8_lossy(&out.stderr)
21174 );
21175 };
21176 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21177 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21178 g(&["push", "-q", "origin", "main"]);
21179 g(&["checkout", "-q", "-b", "feat/own"]);
21180 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21181 g(&["add", "-A"]);
21182 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21183 let reply = tmp.path().join("reply.txt");
21184 std::fs::write(
21185 &reply,
21186 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21187 )
21188 .unwrap();
21189 let mut runner = runner_at(RunStatus::Gating);
21190 let mut state = review_state(&["feat(own): add the own file"]);
21191 state.repo = repo.clone();
21192 state.base_branch = "main".to_owned();
21193 state.candidates[0].branch = "feat/own".to_owned();
21194 state.candidates[0].worktree = repo.clone();
21195 let mut author = spec("alpha");
21196 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21197 author
21198 .env
21199 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21200 state.config.agents = vec![author];
21201 runner.state = state;
21202 SummaryFixture {
21203 _tmp: tmp,
21204 repo,
21205 reply,
21206 runner,
21207 }
21208 }
21209
21210 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21211 let winner = f.runner.state.candidates[0].clone();
21212 f.runner
21213 .guarded_pr_message(&winner, None, true)
21214 .await
21215 .unwrap()
21216 .expect("an English description never parks")
21217 }
21218
21219 fn clean_round_with_minor_findings() -> ReviewRound {
21220 let mut round = passing_round(2, None);
21221 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21222 round.reviews = vec![ReviewRecord {
21223 attempts: 0,
21224 reviewer: 1,
21225 agent: "alpha".into(),
21226 summary: "内部レビュー".into(),
21227 findings: vec![
21228 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21229 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21230 ],
21231 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21232 failed: None,
21233 duration_ms: 0,
21234 }];
21235 round
21236 }
21237
21238 #[tokio::test]
21239 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21240 for review_only in [false, true] {
21241 let mut f = summary_fixture();
21242 if !review_only {
21243 f.runner.state.candidates[0].agent = "alpha".into();
21244 }
21245 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21246 assert!(f.runner.state.reviews[0].clean);
21247 assert!(f.runner.state.reviews[0].fix.is_none());
21248 assert!(f.runner.state.open_findings().is_empty());
21249 assert_eq!(
21250 crate::pr_outcome::required_finding_ids(&f.runner.state),
21251 ["R2-1-1", "R2-1-2"]
21252 );
21253 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21254 let evidence = crate::pr_outcome::summary_evidence(
21255 &f.runner.state,
21256 crate::pr_outcome::FinalHead::At(&head),
21257 );
21258 assert!(
21259 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21260 );
21261 let omitted = posted(&mut f).await;
21263 assert!(omitted.body.contains("No description"));
21264 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21265 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21266 assert!(f.runner.state.change_summary.is_none());
21267 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21268 let described = posted(&mut f).await;
21269 assert!(described.body.contains("improve documentation for callers"));
21270 assert!(described.body.contains("remaining edge case"));
21271 assert!(described.body.contains("## Open review findings"));
21272 assert!(!described.body.contains("未修正"));
21273 }
21274 }
21275
21276 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21277 let mut f = summary_fixture();
21278 if !review_only {
21279 f.runner.state.candidates[0].agent = "alpha".into();
21280 }
21281 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21282 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21283 fixed
21284 .fix
21285 .as_mut()
21286 .unwrap()
21287 .rejected
21288 .push(crate::verdict::Rejection {
21289 id: "R1-1-1".into(),
21290 why: "呼び出されないコードのため辞退".into(),
21291 });
21292 let mut clean = clean_round_with_minor_findings();
21293 clean.verified_head = Some(head.clone());
21294 f.runner.state.reviews = vec![fixed, clean];
21295 ok_gate(&mut f.runner.state, Some(&head));
21296 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21297 f.runner.state.config.agents.push(scripted("beta", reply));
21298 f.runner.state.config.roles.language_judge =
21299 Some(crate::config::AgentChoice::One("beta".into()));
21300 f
21301 }
21302
21303 #[tokio::test]
21304 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21305 for review_only in [false, true] {
21306 let mut f = rewrite_fixture_with_findings(review_only,
21307 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21308 let m = posted(&mut f).await;
21309 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21310 let expected =
21311 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21312 assert!(m.body.contains(&expected));
21313 assert!(m.body.contains("## Review rounds"));
21314 assert!(
21315 m.body
21316 .contains("Gate (`verify.gate`): passed on the final commit")
21317 );
21318 assert!(
21319 m.body
21320 .contains("Tests (`verify.e2e`): passed on the final commit")
21321 );
21322 let c = f.runner.state.change_summary.as_ref().unwrap();
21323 assert!(
21324 m.body
21325 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21326 );
21327 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21328 assert!(crate::pr_outcome::covers_required_findings(
21329 &f.runner.state,
21330 &m.body
21331 ));
21332 assert!(m.body.contains("documentation remains incomplete"));
21333 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21334 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21335 assert!(f.runner.state.github_text.is_none());
21336 }
21337 }
21338
21339 #[tokio::test]
21340 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21341 for review_only in [false, true] {
21342 for reply in [
21343 "TITLE: fix: public description\nA short English description.",
21344 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21345 ] {
21346 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21347 let winner = f.runner.state.candidates[0].clone();
21348 let posted = f
21349 .runner
21350 .guarded_pr_message(&winner, None, true)
21351 .await
21352 .unwrap();
21353 assert!(posted.is_none());
21354 assert!(f.runner.state.parked);
21355 let withheld = f.runner.state.github_text.as_ref().unwrap();
21356 assert!(
21357 withheld.rewrite.is_none(),
21358 "an incomplete rewrite must not be selectable"
21359 );
21360 let q = ask::Questions::open()
21361 .get(withheld.question.as_ref().unwrap())
21362 .unwrap();
21363 assert!(
21364 !q.choices
21365 .contains(&crate::github_text::USE_REWRITE.to_owned())
21366 );
21367 assert!(
21368 q.detail.contains("R1-1-1")
21369 && q.detail.contains("R2-1-1")
21370 && q.detail.contains("R2-1-2")
21371 );
21372 }
21373 }
21374 }
21375
21376 #[tokio::test]
21377 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21378 for review_only in [false, true] {
21379 let mut f = summary_fixture();
21380 if !review_only {
21381 f.runner.state.candidates[0].agent = "alpha".into();
21382 }
21383 f.runner.state.instruction = "内部タスクの説明".into();
21384 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21385 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21386 f.runner.state.reviews = vec![
21387 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21388 round_with_notes(2, Some("二回目の内部修正報告です。")),
21389 passing_round(3, Some(&head)),
21390 ];
21391 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21392 ok_gate(&mut f.runner.state, Some(&head));
21393 let captured = f.repo.join("public-prompt.txt");
21394 let author = &mut f.runner.state.config.agents[0];
21395 author.command = vec![
21396 "sh".into(),
21397 "-c".into(),
21398 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21399 ];
21400 author.env.insert(
21401 "PROMPT_FILE".into(),
21402 captured.to_string_lossy().into_owned(),
21403 );
21404 let m = posted(&mut f).await;
21405 assert!(m.body.contains("Adds `own.txt`"));
21406 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21407 assert!(m.body.contains("R1-1-1"));
21408 assert!(
21409 m.body
21410 .contains("Gate (`verify.gate`): passed on the final commit")
21411 );
21412 assert!(
21413 m.body
21414 .contains("Tests (`verify.e2e`): passed on the final commit")
21415 );
21416 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21417 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21418 let prompt = std::fs::read_to_string(captured).unwrap();
21419 assert!(prompt.contains(&format!(
21420 "git diff {} {head}",
21421 f.runner.state.change_summary.as_ref().unwrap().base
21422 )));
21423 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21424 assert_eq!(
21425 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21426 "二回目の内部修正報告です。"
21427 );
21428 }
21429 }
21430
21431 #[tokio::test]
21432 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21433 for review_only in [false, true] {
21434 let mut f = summary_fixture();
21435 if !review_only {
21436 f.runner.state.candidates[0].agent = "alpha".into();
21437 }
21438 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21439 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21440 std::fs::write(&f.reply, "no usable description").unwrap();
21441 let failed = posted(&mut f).await;
21442 assert!(
21443 failed
21444 .body
21445 .contains("No description of what this change does could be generated")
21446 );
21447 assert!(failed.body.is_ascii());
21448 assert!(f.runner.state.github_text.is_none());
21449 assert!(f.runner.state.change_summary.is_none());
21450 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21451 let evidence = crate::pr_outcome::summary_evidence(
21452 &f.runner.state,
21453 crate::pr_outcome::FinalHead::At(&head),
21454 );
21455 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21456 assert!(
21457 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21458 .exists()
21459 );
21460 f.runner.state =
21461 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21462 std::fs::write(
21463 &f.reply,
21464 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21465 )
21466 .unwrap();
21467 let retried = posted(&mut f).await;
21468 assert!(retried.body.contains("Adds the committed own file."));
21469 assert_eq!(
21470 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21471 "日本語の修正報告"
21472 );
21473 }
21474 }
21475
21476 #[tokio::test]
21477 async fn same_head_verification_and_findings_invalidate_public_prose() {
21478 let mut f = summary_fixture();
21479 posted(&mut f).await;
21480 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21481 let original = f
21482 .runner
21483 .state
21484 .change_summary
21485 .as_ref()
21486 .unwrap()
21487 .evidence
21488 .clone();
21489 ok_gate(&mut f.runner.state, Some(&head));
21490 std::fs::write(
21491 &f.reply,
21492 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21493 )
21494 .unwrap();
21495 let verified = posted(&mut f).await;
21496 assert!(verified.body.contains("now verified"));
21497 assert_ne!(
21498 f.runner.state.change_summary.as_ref().unwrap().evidence,
21499 original
21500 );
21501 let mut round = round_with_notes(1, Some("内部報告"));
21502 round.reviews.push(ReviewRecord {
21503 attempts: 0,
21504 reviewer: 1,
21505 agent: "alpha".into(),
21506 summary: "内部レビュー".into(),
21507 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21508 vote: None,
21509 failed: None,
21510 duration_ms: 0,
21511 });
21512 round
21513 .fix
21514 .as_mut()
21515 .unwrap()
21516 .rejected
21517 .push(crate::verdict::Rejection {
21518 id: "R1-1-1".into(),
21519 why: "呼び出し元から到達できない".into(),
21520 });
21521 f.runner.state.reviews = vec![round];
21522 let omitted = posted(&mut f).await;
21524 assert!(omitted.body.contains("No description"));
21525 assert!(omitted.body.contains("R1-1-1"));
21526 assert!(!omitted.body.contains("到達"));
21527 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21528 let described = posted(&mut f).await;
21529 assert!(described.body.contains("unreachable from callers"));
21530 assert!(described.body.contains("R1-1-1"));
21531 assert!(!described.body.contains("到達"));
21532 let evidence = f
21533 .runner
21534 .state
21535 .change_summary
21536 .as_ref()
21537 .unwrap()
21538 .evidence
21539 .clone();
21540 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21541 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21542 assert!(posted(&mut f).await.body.contains("rationale updated"));
21543 assert_ne!(
21544 f.runner.state.change_summary.as_ref().unwrap().evidence,
21545 evidence
21546 );
21547 f.runner
21549 .state
21550 .change_summary
21551 .as_mut()
21552 .unwrap()
21553 .evidence
21554 .clear();
21555 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21556 assert!(posted(&mut f).await.body.contains("current specification"));
21557 }
21558
21559 #[tokio::test]
21560 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21561 let mut f = summary_fixture();
21562 let m = posted(&mut f).await;
21563 assert_eq!(m.title, "feat(own): add the own file");
21564 assert!(
21565 m.body
21566 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21567 "{}",
21568 m.body
21569 );
21570 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21571 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21572 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21573 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21574 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21575 let kept = f.runner.state.change_summary.as_ref().unwrap();
21576 assert_eq!(
21577 (kept.head.as_str(), kept.base.as_str()),
21578 (head.as_str(), base.as_str())
21579 );
21580 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21581 }
21582
21583 #[tokio::test]
21584 async fn review_only_summary_receives_the_configured_github_language() {
21585 let mut f = summary_fixture();
21586 f.runner.state.config.graph.github_language = "ja".into();
21587 let captured = f.repo.join("summary-prompt.txt");
21588 let author = &mut f.runner.state.config.agents[0];
21589 author.command = vec![
21590 "sh".into(),
21591 "-c".into(),
21592 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21593 ];
21594 author.env.insert(
21595 "PROMPT_FILE".into(),
21596 captured.to_string_lossy().into_owned(),
21597 );
21598 std::fs::write(
21599 &f.reply,
21600 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21601 )
21602 .unwrap();
21603 let m = posted(&mut f).await;
21604 let prompt = std::fs::read_to_string(captured).unwrap();
21605 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21606 assert!(!prompt.contains("in English"));
21607 assert_eq!(m.title, "feat(own): ファイルを追加");
21608 assert!(m.body.contains("own.txt を追加します。"));
21609 }
21610
21611 #[tokio::test]
21612 async fn review_fixes_come_after_the_whole_change_summary() {
21613 let mut f = summary_fixture();
21614 f.runner.state.reviews = vec![
21615 round_with_notes(1, Some("handled the PATH case")),
21616 round_with_notes(2, None),
21617 ];
21618 let m = posted(&mut f).await;
21619 let summary = m.body.find("## Summary").expect("summary");
21620 let fixes = m.body.find("## Review rounds").expect("review rounds");
21621 assert!(summary < fixes, "{}", m.body);
21622 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21623 assert!(m.body.contains("## Final verification"), "{}", m.body);
21624 }
21625
21626 #[tokio::test]
21627 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21628 let mut f = summary_fixture();
21629 let first = posted(&mut f).await;
21630 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21631 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21633 let again = posted(&mut f).await;
21634 assert_eq!((again.title, again.body), (first.title, first.body));
21635
21636 let g = |args: &[&str]| {
21637 let out = std::process::Command::new("git")
21638 .args(args)
21639 .current_dir(&f.repo)
21640 .quiet()
21641 .output()
21642 .expect("spawn git");
21643 assert!(out.status.success(), "git {args:?}");
21644 };
21645 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21646 g(&["add", "-A"]);
21647 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21648 let second = posted(&mut f).await;
21649 let kept = f.runner.state.change_summary.clone().unwrap();
21650 assert_ne!(kept.head, first_head);
21651 assert!(second.body.contains("Not read yet."), "{}", second.body);
21652 assert!(!second.body.contains("so the first summary is read"));
21653 assert_eq!(second.title, "feat(own): other");
21654 }
21655
21656 #[tokio::test]
21657 async fn an_unavailable_summary_is_stated_and_never_invented() {
21658 let mut f = summary_fixture();
21659 std::fs::write(&f.reply, "no title line here").unwrap();
21660 let m = posted(&mut f).await;
21661 assert!(
21662 m.body
21663 .contains("No description of what this change does could be generated"),
21664 "{}",
21665 m.body
21666 );
21667 assert!(m.body.contains("internal reports remain in local records"));
21668 assert!(f.runner.state.change_summary.is_none());
21669 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21670 let winner = f.runner.state.candidates[0].clone();
21672 let preview = f
21673 .runner
21674 .guarded_pr_message(&winner, None, false)
21675 .await
21676 .unwrap()
21677 .unwrap();
21678 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21679 }
21680
21681 #[test]
21682 fn pr_message_review_single_commit_uses_its_subject() {
21683 let state = review_state(&["feat(nats): per-role user"]);
21684 let m = pr_message(&state, 'A');
21685 assert_eq!(m.title, "feat(nats): per-role user");
21686 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21687 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21688 }
21689
21690 #[test]
21691 fn pr_message_review_multi_commit_takes_the_oldest() {
21692 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21693 let m = pr_message(&state, 'A');
21694 assert_eq!(m.title, "feat: the change");
21695 for s in ["feat: the change", "fix: typo", "fix: again"] {
21696 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21697 }
21698 }
21699
21700 #[test]
21701 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21702 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21703 let mut state = review_state(&[first, "fix: later fixup"]);
21704 state.candidates[0].branch = "機能/ログイン".to_owned();
21705 let m = pr_message(&state, 'A');
21706 assert!(
21707 m.title.starts_with("chore: land candidate A of run"),
21708 "{}",
21709 m.title
21710 );
21711 }
21712 }
21713
21714 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21715 BranchFacts {
21716 commits: commits
21717 .iter()
21718 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21719 .collect(),
21720 stat: stat.to_owned(),
21721 }
21722 }
21723
21724 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21725 let mut r = review_round(true, 0, 1, 1, true, true);
21726 r.round = round;
21727 r.fix = notes.map(|n| FixRecord {
21728 agent: "fixer".to_owned(),
21729 addressed: Vec::new(),
21730 rejected: Vec::new(),
21731 notes: n.to_owned(),
21732 committed: true,
21733 failed: None,
21734 duration_ms: 0,
21735 continuation: None,
21736 });
21737 r
21738 }
21739
21740 #[test]
21741 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21742 let state = review_state(&["ignored"]);
21743 let f = facts(
21744 &[
21745 (
21746 "fix(login): resolve PATH on macOS",
21747 "Login shells skip rc files.",
21748 ),
21749 ("fix: address review", ""),
21750 ],
21751 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21752 );
21753 let m = pr_message_with(&state, 'A', Some(&f));
21754 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21755 assert!(
21756 m.body
21757 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
21758 "{}",
21759 m.body
21760 );
21761 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21762 assert!(m.body.contains("## Diff stat"), "{}", m.body);
21763 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21764 for banned in [
21765 "Review the work already",
21766 "no task statement",
21767 "Original task",
21768 ] {
21769 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21770 }
21771 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
21772 }
21773
21774 #[test]
21775 fn pr_message_review_truncates_a_huge_first_commit_body() {
21776 let state = review_state(&["ignored"]);
21777 let f = facts(
21778 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
21779 "s",
21780 );
21781 let m = pr_message_with(&state, 'A', Some(&f));
21782 assert!(m.body.len() < 30_000, "{}", m.body.len());
21783 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
21784 assert!(
21785 m.body.contains("- ... 1 more commit(s)"),
21786 "{}",
21787 m.body.len()
21788 );
21789 assert!(m.body.ends_with("magi:candidate-a\n"));
21790 }
21791
21792 #[test]
21793 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
21794 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
21795 assert_eq!(m.title, "feat: x");
21796 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
21797 assert!(!m.body.contains("Diff stat"), "{}", m.body);
21798 assert!(!m.body.contains("no task statement"), "{}", m.body);
21799 }
21800
21801 #[test]
21802 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
21803 let mut state = review_state(&["日本語の件名"]);
21804 state.candidates[0].branch = "fix/macos-login-path".to_owned();
21805 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
21806 state.candidates[0].branch = "機能/ログイン".to_owned();
21807 assert!(
21808 pr_message(&state, 'A')
21809 .title
21810 .starts_with("chore: land candidate A")
21811 );
21812 }
21813
21814 #[test]
21815 fn pr_message_review_fixes_survive_a_clean_final_round() {
21816 let mut state = review_state(&["feat: x"]);
21817 state.reviews = vec![
21818 round_with_notes(1, Some("handled the PATH case")),
21819 round_with_notes(2, None),
21820 ];
21821 let body = pr_message(&state, 'A').body;
21822 assert!(
21823 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
21824 "{body}"
21825 );
21826 assert!(body.contains("### Round 1"), "{body}");
21827
21828 state.reviews = vec![
21829 round_with_notes(1, Some("first fix")),
21830 round_with_notes(2, Some("")),
21831 round_with_notes(3, Some("second fix")),
21832 round_with_notes(4, None),
21833 ];
21834 let body = pr_message(&state, 'A').body;
21835 assert!(
21836 body.contains("### Round 1") && !body.contains("first fix"),
21837 "{body}"
21838 );
21839 assert!(
21840 body.contains("### Round 3") && !body.contains("second fix"),
21841 "{body}"
21842 );
21843 assert!(body.contains("### Round 2"), "{body}");
21844 }
21845
21846 const FINAL: &str = "abcdef1234567890";
21847
21848 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
21849 let mut r = round_with_notes(round, None);
21850 r.verified_head = verified.map(str::to_owned);
21851 r
21852 }
21853
21854 fn ok_gate(state: &mut RunState, head: Option<&str>) {
21855 state.gate = vec![CommandOutcome {
21856 command: "gate".to_owned(),
21857 code: Some(0),
21858 output_tail: String::new(),
21859 duration_ms: 0,
21860 resource_blocked: false,
21861 }];
21862 state.gate_ran = true;
21863 state.gate_head = head.map(str::to_owned);
21864 }
21865
21866 fn body_at(state: &RunState) -> String {
21867 pr_message_raw(
21868 state,
21869 'A',
21870 None,
21871 crate::pr_outcome::FinalHead::At(FINAL),
21872 ChangeNote::NotAsked,
21873 )
21874 .body
21875 }
21876
21877 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
21878
21879 #[test]
21880 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
21881 let mut state = review_state(&["feat: x"]);
21882 state.reviews = vec![
21883 round_with_notes(1, Some(SANDBOXED)),
21884 passing_round(2, Some(FINAL)),
21885 ];
21886 ok_gate(&mut state, Some(FINAL));
21887 let body = body_at(&state);
21888 assert!(!body.contains("Handled the PATH case."), "{body}");
21889 assert!(!body.contains("could not commit"), "{body}");
21890 assert!(!body.contains("could not run"), "{body}");
21891 assert!(
21892 body.contains("The fix is committed on the branch"),
21893 "{body}"
21894 );
21895 assert!(!body.contains("sandbox"), "{body}");
21896 assert!(
21897 body.contains("Tests (`verify.e2e`): passed on the final commit"),
21898 "{body}"
21899 );
21900 assert!(
21901 body.contains("Gate (`verify.gate`): passed on the final commit"),
21902 "{body}"
21903 );
21904 assert!(
21906 state.reviews[0]
21907 .fix
21908 .as_ref()
21909 .unwrap()
21910 .notes
21911 .contains("could not commit")
21912 );
21913 }
21914
21915 #[test]
21916 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
21917 let mut state = review_state(&["feat: x"]);
21918 let mut r1 = round_with_notes(1, Some(SANDBOXED));
21919 r1.fix.as_mut().unwrap().committed = false;
21920 let mut r2 = passing_round(2, Some(FINAL));
21921 r2.e2e[0].code = Some(1);
21922 state.reviews = vec![r1, r2];
21923 let body = body_at(&state);
21924 assert!(!body.contains("could not commit"), "{body}");
21925 assert!(!body.contains("could not run"), "{body}");
21926 assert!(body.contains("This round produced no commit"), "{body}");
21927 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
21928 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
21929 assert!(!body.contains("passed"), "{body}");
21930 }
21931
21932 #[test]
21933 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
21934 let mut state = review_state(&["feat: x"]);
21935 state.reviews = vec![
21936 round_with_notes(1, Some(SANDBOXED)),
21937 passing_round(2, Some("0123456789abcdef")),
21938 ];
21939 ok_gate(&mut state, Some("0123456789abcdef"));
21940 let body = body_at(&state);
21941 assert!(!body.contains("HTTP tests could not run"), "{body}");
21942 assert!(!body.contains("could not commit"), "{body}");
21944 assert!(
21945 body.contains("passed on an earlier commit (`0123456`)"),
21946 "{body}"
21947 );
21948 assert!(!body.contains("passed on the final commit"), "{body}");
21949
21950 ok_gate(&mut state, None);
21952 let body = body_at(&state);
21953 assert!(
21954 body.contains("passed on a commit that was not recorded"),
21955 "{body}"
21956 );
21957 }
21958
21959 #[test]
21960 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
21961 let state = state_with_summary(
21962 "add retries to the client",
21963 "TITLE: feat: retries\n\nDid it.",
21964 );
21965 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
21966 assert_eq!(m.title, "feat: retries");
21967 assert!(
21968 m.body.contains("<summary>Original task</summary>"),
21969 "{}",
21970 m.body
21971 );
21972 assert!(!m.body.contains("Commits under review"), "{}", m.body);
21973 assert!(
21974 m.body
21975 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
21976 "{}",
21977 m.body
21978 );
21979 }
21980
21981 #[test]
21982 fn pr_message_review_bounds_a_long_english_subject() {
21983 let long = format!("feat: {}", "word ".repeat(100));
21984 let m = pr_message(&review_state(&[&long]), 'A');
21985 assert!(m.title.starts_with("feat: word"), "{}", m.title);
21986 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
21987 }
21988
21989 #[test]
21990 fn pr_message_implementation_run_is_unchanged_by_review_support() {
21991 let state = state_with_summary("add retries\n\ndetails", "- did some things");
21992 let m = pr_message(&state, 'A');
21993 assert_eq!(m.title, "add retries");
21994 assert!(m.body.contains("<summary>Original task</summary>"));
21995 assert!(!m.body.contains("Commits under review"));
21996 assert_eq!(landing_subject_source(&state), state.instruction);
21997 }
21998
21999 #[test]
22000 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22001 let state = review_state(&["feat: the change", "fix: typo"]);
22002 let source = landing_subject_source(&state);
22003 assert_eq!(land::merge_subject("", &source), "feat: the change");
22004 assert_eq!(
22005 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22006 "feat: the change"
22007 );
22008 assert_eq!(
22010 land::merge_subject("feat: renamed by hand", &source),
22011 "feat: renamed by hand"
22012 );
22013 let blank = review_state(&["日本語"]);
22014 assert!(
22015 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22016 );
22017 }
22018
22019 #[test]
22020 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22021 let state = review_state(&["feat: the change"]);
22022 let source = landing_subject_source(&state);
22023 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22024 assert_eq!(
22025 land::merge_subject(landing_title(&state, old), &source),
22026 "feat: the change"
22027 );
22028 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22029 let task = state_with_summary("add retries", "");
22030 assert_eq!(landing_title(&task, old), old);
22031 }
22032
22033 #[test]
22034 fn pr_message_describes_the_change_not_the_task() {
22035 let state = state_with_summary(
22036 "今回やってほしいこと: results projector を直す",
22037 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22038 );
22039 let m = pr_message(&state, 'A');
22040 assert_eq!(m.title, "fix(web): batch the runs list reads");
22041 assert!(
22042 m.body.starts_with("## Summary\n\n- reads run.json once"),
22043 "{}",
22044 m.body
22045 );
22046 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22047 let task_at = m.body.find("今回やってほしいこと").unwrap();
22048 let details_at = m.body.find("<details>").unwrap();
22049 assert!(
22050 details_at < task_at,
22051 "the task lives inside <details>: {}",
22052 m.body
22053 );
22054 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22055 assert!(m.body.contains("magi:candidate-a"));
22056 }
22057
22058 #[test]
22059 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22060 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22061 let m = pr_message(&state, 'A');
22062 assert_eq!(m.title, "add retries");
22063 assert!(
22064 m.body.contains("## Summary\n\n- did some things"),
22065 "{}",
22066 m.body
22067 );
22068
22069 let none = RunState::new(
22070 PathBuf::from("/repo"),
22071 "main".to_owned(),
22072 "abc1234".to_owned(),
22073 "add retries".to_owned(),
22074 Config::default(),
22075 );
22076 let m = pr_message(&none, 'A');
22077 assert_eq!(m.title, "add retries");
22078 assert!(!m.body.contains("## Summary"), "{}", m.body);
22079 }
22080
22081 #[test]
22082 fn pr_message_refuses_the_candidate_commit_subject() {
22083 for bad in [
22084 "TITLE: magi: candidate A (uncommitted work)",
22085 "TITLE: chore: stuff (uncommitted work)",
22086 "TITLE: ",
22087 ] {
22088 let state = state_with_summary("add retries", bad);
22089 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22090 }
22091 }
22092
22093 #[test]
22094 fn pr_message_bounds_a_very_long_task_and_title() {
22095 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22096 let state = state_with_summary(&long, "- nothing");
22097 let m = pr_message(&state, 'A');
22098 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22099 assert!(!m.title.contains('\n'));
22100
22101 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22102 let m = pr_message(&state, 'A');
22103 assert!(m.title.starts_with("feat: "));
22104 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22105 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22106 }
22107
22108 fn long_title_of(instruction: &str) -> String {
22109 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22110 }
22111
22112 #[test]
22113 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22114 let first = "Make the landing path keep a readable title for long tasks";
22115 let line = format!(
22116 "{first}. {}",
22117 "And then keep going with more words ".repeat(20)
22118 );
22119 let t = long_title_of(&line);
22120 assert_eq!(t, first);
22121 assert!(!t.starts_with("chore: land"));
22122 }
22123
22124 #[test]
22125 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22126 let line = "word ".repeat(200);
22127 let t = long_title_of(&line);
22128 assert!(t.ends_with("word..."), "{t}");
22129 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22130 }
22131
22132 #[test]
22133 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22134 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22135 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22136 }
22137 }
22138
22139 #[test]
22140 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22141 let mut state = state_with_summary("再試行を追加する", "- nothing");
22142 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22143 state.config.graph.github_language = "ja".into();
22144 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22145 }
22146
22147 #[test]
22148 fn pr_message_title_limit_is_exact() {
22149 let at = "a".repeat(PR_TITLE_MAX);
22150 assert_eq!(long_title_of(&at), at);
22151 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22152 assert!(over.ends_with("..."), "{over}");
22153 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22154 }
22155
22156 #[test]
22157 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22158 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22159 let t = long_title_of(&line);
22160 assert!(t.ends_with("..."), "{t}");
22161 assert!(t.is_ascii(), "{t}");
22162 }
22163
22164 #[test]
22165 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22166 let line = format!(
22167 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22168 "plus more filler words ".repeat(20)
22169 );
22170 let t = long_title_of(&line);
22171 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22172 }
22173
22174 #[test]
22175 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22176 let line = format!("\n\n# {}", "title words ".repeat(40));
22177 let state = state_with_summary(&line, "- nothing");
22178 let m = pr_message(&state, 'A');
22179 assert!(m.title.starts_with("title words"), "{}", m.title);
22180 assert_eq!(
22181 land::merge_subject(&m.title, &landing_subject_source(&state)),
22182 m.title
22183 );
22184 assert_eq!(
22186 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22187 "feat: renamed by hand"
22188 );
22189 }
22190
22191 #[tokio::test]
22192 async fn github_text_rewrite_is_bounded_and_falls_back() {
22193 crate::run::pin_test_home();
22194 for (reply, accepted) in [
22195 (
22196 "TITLE: fix: retries\nAdd retries for failed requests.",
22197 true,
22198 ),
22199 (
22200 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22201 false,
22202 ),
22203 (
22204 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22205 true,
22206 ),
22207 (
22208 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22209 true,
22210 ),
22211 (
22212 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22213 true,
22214 ),
22215 (
22216 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22217 false,
22218 ),
22219 ("TITLE: fix: retries\n", false),
22220 ] {
22221 let dir = tempfile::tempdir().unwrap();
22222 let mut runner = runner_at(RunStatus::Gating);
22223 runner.state = state_with_summary(
22224 "add retries",
22225 "TITLE: fix: retries\n日本語の説明を書きます。",
22226 );
22227 runner.state.repo = dir.path().to_owned();
22228 runner.state.candidates[0].worktree = dir.path().to_owned();
22229 let mut author = spec("alpha");
22230 author.command = vec![
22231 "sh".into(),
22232 "-c".into(),
22233 "printf '%s' \"$REWRITE_REPLY\"".into(),
22234 ];
22235 author.env.insert("REWRITE_REPLY".into(), reply.into());
22236 runner.state.config.agents = vec![author];
22237 seed_public_summary(&mut runner);
22238 let winner = runner.state.candidates[0].clone();
22239 let first = runner
22240 .guarded_pr_message(&winner, None, true)
22241 .await
22242 .unwrap();
22243 let message = if accepted {
22244 first.expect("a passing rewrite never asks")
22245 } else {
22246 assert!(first.is_none());
22248 assert!(runner.state.parked);
22249 let store = ask::Questions::open();
22250 let asked: Vec<_> = store
22251 .list()
22252 .into_iter()
22253 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22254 .collect();
22255 assert_eq!(asked.len(), 1);
22256 assert!(!asked[0].detail.contains("secret"));
22257 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22258 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22259 assert_eq!(
22260 asked[0]
22261 .choices
22262 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22263 rewrite.is_some()
22264 );
22265 store
22268 .update(&asked[0].id, |q| {
22269 q.abandon("test");
22270 Ok(())
22271 })
22272 .unwrap();
22273 let again = runner
22274 .guarded_pr_message(&winner, None, true)
22275 .await
22276 .unwrap()
22277 .expect("an abandoned question falls back");
22278 let n = store
22279 .list()
22280 .into_iter()
22281 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22282 .count();
22283 assert_eq!(n, 1, "asked once per rejected text");
22284 again
22285 };
22286 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22287 assert!(!message.body.contains("secret"));
22288 assert!(!message.body.contains("prefix"));
22289 assert!(!message.title.contains("secret"));
22290 assert_eq!(
22291 message.body.contains("Add retries for failed requests."),
22292 accepted
22293 );
22294 assert_eq!(
22295 runner.state.seats["github-text-rewrite"].turns, 1,
22296 "only one rewrite invocation"
22297 );
22298 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22299 if accepted {
22300 assert!(runner.state.github_text.is_none());
22301 assert!(!runner.state.parked);
22302 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22303 assert!(message.body.contains("## Final verification"));
22304 assert!(message.body.contains("Evidence: final diff"));
22305 assert!(
22306 message
22307 .body
22308 .contains(&format!("magi:run/{}", runner.state.id))
22309 );
22310 }
22311 if !accepted {
22312 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22313 assert!(
22314 message
22315 .body
22316 .contains(&format!("magi:run/{}", runner.state.id))
22317 );
22318 }
22319 }
22320 }
22321
22322 fn scripted(id: &str, reply: &str) -> AgentSpec {
22323 let mut a = spec(id);
22324 a.command = vec![
22325 "sh".into(),
22326 "-c".into(),
22327 "printf '%s' \"$REWRITE_REPLY\"".into(),
22328 ];
22329 a.env.insert("REWRITE_REPLY".into(), reply.into());
22330 a
22331 }
22332
22333 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22334 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22335
22336 fn seed_public_summary(runner: &mut Runner) {
22338 let repo = &runner.state.repo;
22339 let g = |args: &[&str]| {
22340 let out = std::process::Command::new("git")
22341 .args(args)
22342 .current_dir(repo)
22343 .quiet()
22344 .output()
22345 .unwrap();
22346 assert!(out.status.success(), "git {args:?}");
22347 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22348 };
22349 g(&["init", "-q", "-b", "main"]);
22350 g(&[
22351 "-c",
22352 "user.name=Test",
22353 "-c",
22354 "user.email=test@example.invalid",
22355 "commit",
22356 "-q",
22357 "--allow-empty",
22358 "-m",
22359 "feat: fixture",
22360 ]);
22361 let head = g(&["rev-parse", "HEAD"]);
22362 g(&["update-ref", "refs/remotes/origin/main", &head]);
22363 let evidence = crate::pr_outcome::summary_evidence(
22364 &runner.state,
22365 crate::pr_outcome::FinalHead::At(&head),
22366 );
22367 let generated = &runner.state.candidates[0].summary;
22368 runner.state.change_summary = Some(crate::run::ChangeSummary {
22369 base: head.clone(),
22370 head,
22371 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22372 title: summary_title(generated).unwrap(),
22373 body: summary_without_title(generated),
22374 });
22375 }
22376
22377 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22378 let mut runner = runner_at(RunStatus::Gating);
22379 runner.state = state_with_summary("add retries", JA_SUMMARY);
22380 if review_only {
22381 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22382 }
22383 runner.state.repo = dir.to_owned();
22384 runner.state.candidates[0].worktree = dir.to_owned();
22385 runner.state.config.agents = agents;
22386 seed_public_summary(&mut runner);
22387 runner
22388 }
22389
22390 fn gate_questions(runner: &Runner) -> usize {
22391 ask::Questions::open()
22392 .list()
22393 .into_iter()
22394 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22395 .count()
22396 }
22397
22398 #[tokio::test]
22399 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22400 crate::run::pin_test_home();
22401 let dir = tempfile::tempdir().unwrap();
22402 let mut runner = rewrite_runner(
22403 dir.path(),
22404 true,
22405 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22406 );
22407 runner.state.config.roles.language_judge =
22408 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22409 let winner = runner.state.candidates[0].clone();
22410 let msg = runner
22411 .guarded_pr_message(&winner, None, true)
22412 .await
22413 .unwrap()
22414 .expect("an English rewrite posts without asking");
22415 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22416 assert!(msg.body.contains("<details>kept</details>"));
22417 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22418 assert_eq!(gate_questions(&runner), 0);
22419 assert!(runner.state.github_text.is_none());
22420 assert!(!runner.state.seats.contains_key("impl-A"));
22421 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22422 }
22423
22424 #[tokio::test]
22425 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22426 crate::run::pin_test_home();
22427 let dir = tempfile::tempdir().unwrap();
22428 let mut runner = rewrite_runner(
22429 dir.path(),
22430 false,
22431 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22432 );
22433 runner.state.config.roles.language_judge =
22434 Some(crate::config::AgentChoice::One("beta".into()));
22435 runner.state.seats.insert(
22436 "impl-A".into(),
22437 SeatState::new("impl-A", "alpha", runner.state.seed),
22438 );
22439 let winner = runner.state.candidates[0].clone();
22440 let msg = runner
22441 .guarded_pr_message(&winner, None, true)
22442 .await
22443 .unwrap()
22444 .expect("rewritten");
22445 assert!(msg.body.contains("<details>kept</details>"));
22446 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22447 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22448 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22449 }
22450
22451 #[tokio::test]
22452 async fn rewrite_reuses_a_resumable_implementer_session() {
22453 crate::run::pin_test_home();
22454 let dir = tempfile::tempdir().unwrap();
22455 let mut runner = rewrite_runner(
22456 dir.path(),
22457 false,
22458 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22459 );
22460 runner.state.config.roles.language_judge =
22461 Some(crate::config::AgentChoice::One("beta".into()));
22462 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22463 seat.turns = 1;
22464 runner.state.seats.insert("impl-A".into(), seat);
22465 let winner = runner.state.candidates[0].clone();
22466 runner
22467 .guarded_pr_message(&winner, None, true)
22468 .await
22469 .unwrap()
22470 .expect("rewritten");
22471 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22472 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22473 }
22474
22475 #[tokio::test]
22476 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22477 crate::run::pin_test_home();
22480 let dir = tempfile::tempdir().unwrap();
22481 let mut exhausted = spec("alpha");
22482 exhausted.command = vec!["false".into()];
22483 let mut runner = rewrite_runner(
22484 dir.path(),
22485 false,
22486 vec![exhausted, scripted("beta", EN_REWRITE)],
22487 );
22488 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22490 "alpha".into(),
22491 "beta".into(),
22492 ]));
22493 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22494 seat.turns = 1;
22495 runner.state.seats.insert("impl-A".into(), seat);
22496 let winner = runner.state.candidates[0].clone();
22497 let msg = runner
22498 .guarded_pr_message(&winner, None, true)
22499 .await
22500 .unwrap()
22501 .expect("the role rewrites after the implementer fails");
22502 assert!(msg.body.contains("`src/net.rs`"));
22503 assert_eq!(
22504 runner.state.seats["impl-A"].turns, 2,
22505 "the implementer was tried once"
22506 );
22507 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22508 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22509 }
22510
22511 #[tokio::test]
22512 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22513 crate::run::pin_test_home();
22514 let dir = tempfile::tempdir().unwrap();
22515 let mut broken = spec("bad");
22516 broken.command = vec!["false".into()];
22517 let mut runner = rewrite_runner(
22518 dir.path(),
22519 true,
22520 vec![
22521 broken,
22522 scripted("noformat", "Add retries."),
22523 scripted("good", EN_REWRITE),
22524 ],
22525 );
22526 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22527 "bad".into(),
22528 "noformat".into(),
22529 "good".into(),
22530 ]));
22531 let winner = runner.state.candidates[0].clone();
22532 let msg = runner
22533 .guarded_pr_message(&winner, None, true)
22534 .await
22535 .unwrap()
22536 .expect("the third agent rewrites");
22537 assert!(msg.body.contains("`src/net.rs`"));
22538 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22539 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22540 }
22541
22542 #[tokio::test]
22543 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22544 crate::run::pin_test_home();
22545 for (agents, role, saved) in [
22547 (
22548 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22549 Some("beta"),
22550 true,
22551 ),
22552 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22553 (Vec::new(), None, false),
22554 ] {
22555 let dir = tempfile::tempdir().unwrap();
22556 let mut runner = rewrite_runner(dir.path(), true, agents);
22557 runner.state.config.roles.language_judge =
22558 role.map(|r| crate::config::AgentChoice::One(r.into()));
22559 let winner = runner.state.candidates[0].clone();
22560 let first = runner
22561 .guarded_pr_message(&winner, None, true)
22562 .await
22563 .unwrap();
22564 assert!(first.is_none(), "withheld text parks the run");
22565 assert!(runner.state.parked);
22566 assert_eq!(gate_questions(&runner), 1);
22567 let g = runner.state.github_text.as_ref().unwrap();
22568 assert_eq!(g.rewrite.is_some(), saved);
22569 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22570 }
22571 }
22572
22573 #[tokio::test]
22574 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22575 crate::run::pin_test_home();
22576 let dir = tempfile::tempdir().unwrap();
22577 let mut runner = runner_at(RunStatus::Gating);
22578 runner.state =
22579 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22580 runner.state.repo = dir.path().to_owned();
22581 runner.state.candidates[0].worktree = dir.path().to_owned();
22582 let mut author = spec("alpha");
22583 author.command = vec![
22584 "sh".into(),
22585 "-c".into(),
22586 "printf '%s' \"$REWRITE_REPLY\"".into(),
22587 ];
22588 author.env.insert(
22589 "REWRITE_REPLY".into(),
22590 format!(
22591 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22592 "English sentence. ".repeat(600)
22593 ),
22594 );
22595 runner.state.config.agents = vec![author];
22596 seed_public_summary(&mut runner);
22597 let winner = runner.state.candidates[0].clone();
22598 assert!(
22599 runner
22600 .guarded_pr_message(&winner, None, true)
22601 .await
22602 .unwrap()
22603 .is_none()
22604 );
22605 let record = runner.state.github_text.clone().unwrap();
22606 let rewrite = record.rewrite.clone().unwrap();
22607 assert_eq!(rewrite.failed, ["body-language"]);
22608 assert!(
22609 rewrite
22610 .body
22611 .ends_with(&format!("magi:run/{}", runner.state.id))
22612 );
22613 let store = ask::Questions::open();
22614 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22615 assert!(
22616 q.choices
22617 .contains(&crate::github_text::USE_REWRITE.to_owned())
22618 );
22619 for label in [
22620 "ORIGINAL",
22621 "FALLBACK",
22622 "REWRITE",
22623 "Rules still failed: body-language",
22624 "truncated",
22625 ] {
22626 assert!(q.detail.contains(label), "{label}");
22627 }
22628 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22629 &runner.state,
22630 &crate::github_text::artifact_name(&record.fingerprint),
22631 ))
22632 .unwrap();
22633 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22634 runner.state =
22636 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22637 store
22638 .update(&q.id, |q| {
22639 q.say("まだ日本語")?;
22640 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22641 })
22642 .unwrap();
22643 assert!(
22644 runner
22645 .guarded_pr_message(&winner, None, true)
22646 .await
22647 .unwrap()
22648 .is_none()
22649 );
22650 let retry = runner.state.github_text.as_ref().unwrap();
22651 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22652 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22653 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22654 assert!(
22655 q.posting_comparison
22656 .as_ref()
22657 .unwrap()
22658 .matches_detail(&q.detail)
22659 );
22660 store
22661 .update(&q.id, |q| {
22662 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22663 })
22664 .unwrap();
22665 let message = runner
22666 .guarded_pr_message(&winner, None, true)
22667 .await
22668 .unwrap()
22669 .unwrap();
22670 assert_eq!(message.title, rewrite.title);
22671 assert_eq!(message.body, rewrite.body);
22672 assert_eq!(
22673 runner.state.github_text.as_ref().unwrap().choice,
22674 Some(GithubTextChoice::Rewrite)
22675 );
22676 let again = runner
22677 .guarded_pr_message(&winner, None, true)
22678 .await
22679 .unwrap()
22680 .unwrap();
22681 assert_eq!(again.title, message.title);
22682 assert_eq!(again.body, message.body);
22683 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22684 assert!(
22685 !runner
22686 .state
22687 .events
22688 .iter()
22689 .any(|e| e.message.contains(&rewrite.title)
22690 || e.message.contains("English sentence."))
22691 );
22692 }
22693
22694 #[tokio::test]
22695 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22696 crate::run::pin_test_home();
22697 let dir = tempfile::tempdir().unwrap();
22698 let mut runner = runner_at(RunStatus::Gating);
22699 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22700 runner.state.repo = dir.path().to_owned();
22701 runner.state.candidates[0].worktree = dir.path().to_owned();
22702 seed_public_summary(&mut runner);
22703 let winner = runner.state.candidates[0].clone();
22704 assert!(
22705 runner
22706 .guarded_pr_message(&winner, None, true)
22707 .await
22708 .unwrap()
22709 .is_none()
22710 );
22711 let record = runner.state.github_text.as_ref().unwrap();
22712 assert!(record.rewrite.is_none());
22713 let mut old = serde_json::to_value(record).unwrap();
22715 old.as_object_mut().unwrap().remove("rewrite");
22716 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22717 assert!(old.rewrite.is_none());
22718 let id = record.question.clone().unwrap();
22719 ask::Questions::open()
22720 .update(&id, |q| {
22721 q.choices.push(crate::github_text::USE_REWRITE.into());
22722 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22723 })
22724 .unwrap();
22725 let message = runner
22726 .guarded_pr_message(&winner, None, true)
22727 .await
22728 .unwrap()
22729 .unwrap();
22730 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22731 assert_eq!(
22732 runner.state.github_text.as_ref().unwrap().choice,
22733 Some(GithubTextChoice::Fallback)
22734 );
22735 }
22736
22737 #[tokio::test]
22738 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22739 crate::run::pin_test_home();
22740 let dir = tempfile::tempdir().unwrap();
22741 let mut runner = runner_at(RunStatus::Gating);
22742 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22743 runner.state.repo = dir.path().to_owned();
22744 runner.state.candidates[0].worktree = dir.path().to_owned();
22745 seed_public_summary(&mut runner);
22746 let winner = runner.state.candidates[0].clone();
22747 assert!(
22748 runner
22749 .guarded_pr_message(&winner, None, true)
22750 .await
22751 .unwrap()
22752 .is_none()
22753 );
22754 let store = ask::Questions::open();
22755 let find = |run: &str| {
22756 store
22757 .list()
22758 .into_iter()
22759 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22760 .unwrap()
22761 };
22762 let run = runner.state.id.clone();
22763 let q = find(&run);
22764 let fp = runner
22765 .state
22766 .github_text
22767 .as_ref()
22768 .unwrap()
22769 .fingerprint
22770 .clone();
22771 let name = crate::github_text::artifact_name(&fp);
22772 let path = crate::run::artifact_path(&runner.state, &name);
22773 assert!(q.detail.contains(&path.display().to_string()));
22774 assert!(q.detail.contains("title-language: title"));
22775 let saved = std::fs::read_to_string(&path).expect("artifact written");
22776 assert!(saved.contains("日本語のタイトル"));
22777 assert!(q.cwd.is_none());
22778 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
22779 let say = |id: &str, text: &str, choice: &str| {
22781 store
22782 .update(id, |q| {
22783 q.thread.push(ask::Turn {
22784 who: ask::Who::Operator,
22785 body: text.to_owned(),
22786 at: jiff::Timestamp::now(),
22787 note: None,
22788 });
22789 q.answer(ask::Answer::Choice(choice.to_owned()))
22790 })
22791 .unwrap();
22792 };
22793 say(&q.id, "まだ日本語", "use my text");
22794 assert!(
22795 runner
22796 .guarded_pr_message(&winner, None, true)
22797 .await
22798 .unwrap()
22799 .is_none()
22800 );
22801 let second = find(&run);
22802 assert_ne!(second.id, q.id);
22803 assert!(second.detail.contains("did not pass"));
22804 assert!(second.detail.contains("日本語のタイトル"));
22807 assert!(!second.detail.contains("まだ日本語"));
22808 assert!(second.detail.contains(&path.display().to_string()));
22809 say(&second.id, "fix: retry failed requests", "use my text");
22810 let message = runner
22811 .guarded_pr_message(&winner, None, true)
22812 .await
22813 .unwrap()
22814 .expect("a vetted replacement is posted");
22815 assert_eq!(message.title, "fix: retry failed requests");
22816 assert_eq!(
22817 runner
22818 .state
22819 .github_text
22820 .as_ref()
22821 .unwrap()
22822 .chosen_title
22823 .as_deref(),
22824 Some("fix: retry failed requests")
22825 );
22826 assert!(runner.state.github_text.as_ref().unwrap().resolved);
22827 let again = runner
22829 .guarded_pr_message(&winner, None, true)
22830 .await
22831 .unwrap()
22832 .unwrap();
22833 assert_eq!(again.title, message.title);
22834 assert_eq!(
22835 store
22836 .list()
22837 .into_iter()
22838 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
22839 .count(),
22840 2
22841 );
22842 }
22843
22844 #[test]
22845 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
22846 let mut state = state_with_summary(
22850 "add retries",
22851 "TITLE: fix(web): batch reads\n- reads run.json once",
22852 );
22853 state.config.graph.language = "ja".to_owned();
22854 let m = pr_message(&state, 'A');
22855 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
22856 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
22857
22858 let task = "今回やってほしいこと: results projector を直す";
22861 let mut state = state_with_summary(task, "- no title line");
22862 state.config.graph.language = "ja".to_owned();
22863 let m = pr_message(&state, 'A');
22864 assert_eq!(
22865 m.title,
22866 format!("chore: land candidate A of run {}", state.id)
22867 );
22868 assert!(
22869 m.body.contains(&format!(
22870 "<summary>Original task</summary>\n\n{task}\n\n</details>"
22871 )),
22872 "{}",
22873 m.body
22874 );
22875 }
22876
22877 #[test]
22878 fn pr_message_scrubs_home_paths_and_addresses() {
22879 let state = state_with_summary(
22880 "fix it in /Users/someone/src/x",
22881 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
22882 );
22883 let m = pr_message(&state, 'A');
22884 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
22885 assert!(!m.body.contains(leak), "{}", m.body);
22886 }
22887 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
22888 }
22889
22890 #[test]
22891 fn pr_message_survives_a_task_that_closes_details() {
22892 let state = state_with_summary("a </details> b", "TITLE: fix: x");
22893 let m = pr_message(&state, 'A');
22894 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
22895 }
22896
22897 #[test]
22898 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
22899 let cmd = manual_merge_command(
22900 MergeStyle::Squash,
22901 Path::new("/repo"),
22902 "b",
22903 "fix: \"quoted\" $(x) `y`\n\nbody",
22904 );
22905 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
22906 }
22907
22908 #[test]
22909 fn manual_merge_command_matches_the_configured_style() {
22910 let repo = Path::new("/repo");
22911 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
22912
22913 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
22914 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
22915
22916 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
22917 assert_eq!(
22918 squash,
22919 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
22920 \"Merge magi run 0832 (candidate A)\""
22921 );
22922
22923 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
22924 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
22925 }
22926
22927 #[test]
22928 fn a_nudge_gets_a_quarter_of_the_budget() {
22929 assert_eq!(retry_budget(secs(1200), true), secs(300));
22931 assert_eq!(retry_budget(secs(3600), true), secs(900));
22932 }
22933
22934 #[test]
22935 fn a_resent_prompt_keeps_the_whole_budget() {
22936 assert_eq!(retry_budget(secs(1200), false), secs(1200));
22939 assert_eq!(retry_budget(secs(60), false), secs(60));
22940 }
22941
22942 #[test]
22943 fn the_floor_never_exceeds_the_original_budget() {
22944 assert_eq!(retry_budget(secs(60), true), secs(60));
22948 assert_eq!(retry_budget(secs(480), true), secs(120));
22949 assert_eq!(retry_budget(secs(0), true), secs(0));
22950 }
22951
22952 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
22953 agent::CommandEvidence {
22954 id: "item1".to_owned(),
22955 description: "cargo test".to_owned(),
22956 exit_code,
22957 result_summary: String::new(),
22958 source: "codex".to_owned(),
22959 }
22960 }
22961
22962 #[test]
22963 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
22964 assert!(!has_unconfirmed_command(&[]));
22968 }
22969
22970 #[test]
22971 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
22972 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
22976 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
22977 assert!(!has_unconfirmed_command(&[
22978 evidence(Some(0)),
22979 evidence(Some(101))
22980 ]));
22981 }
22982
22983 #[test]
22984 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
22985 assert!(has_unconfirmed_command(&[
22986 evidence(Some(0)),
22987 evidence(None)
22988 ]));
22989 }
22990
22991 #[test]
22992 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
22993 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
22994 assert_eq!(
22995 verified_noop_claim(true, &[], text).as_deref(),
22996 Some("already fixed by b32cfc4, on main.")
22997 );
22998 }
22999
23000 #[test]
23001 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23002 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23005 assert!(verified_noop_claim(false, &[], text).is_none());
23006 }
23007
23008 #[test]
23009 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23010 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23011 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23012 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23014 }
23015
23016 #[test]
23017 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23018 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23019 }
23020
23021 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23024 runner.state.candidates = shape
23025 .iter()
23026 .enumerate()
23027 .map(|(i, &(empty, verified))| Candidate {
23028 index: i,
23029 label: (b'A' + i as u8) as char,
23030 agent: "sonnet".to_owned(),
23031 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23032 worktree: PathBuf::from(format!("/wt/{i}")),
23033 summary: String::new(),
23034 stat: String::new(),
23035 files: 0,
23036 commits: 0,
23037 empty,
23038 failed: None,
23039 verified_noop: verified.map(str::to_owned),
23040 duration_ms: 0,
23041 folded: false,
23042 })
23043 .collect();
23044 }
23045
23046 #[test]
23047 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23048 ask_test_home();
23049 let mut runner = runner_at(RunStatus::Implementing);
23050 set_candidates(
23051 &mut runner,
23052 &[
23053 (true, Some("already on main at b32cfc4")),
23054 (true, Some("same fix, see the existing test")),
23055 ],
23056 );
23057
23058 runner
23059 .after_implement()
23060 .expect("a verified no-op is not an error");
23061
23062 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23063 }
23064
23065 #[test]
23066 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23067 ask_test_home();
23068 let mut runner = runner_at(RunStatus::Implementing);
23069 set_candidates(
23073 &mut runner,
23074 &[(true, Some("already on main at b32cfc4")), (true, None)],
23075 );
23076
23077 let err = runner
23078 .after_implement()
23079 .expect_err("an unverified empty candidate must still fail the run");
23080
23081 assert!(
23082 err.to_string().contains("no candidate produced a change"),
23083 "{err}"
23084 );
23085 assert_eq!(runner.state.status, RunStatus::Failed);
23086 }
23087
23088 #[test]
23089 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23090 ask_test_home();
23091 let mut runner = runner_at(RunStatus::Implementing);
23092 set_candidates(&mut runner, &[(true, None), (true, None)]);
23093
23094 let err = runner
23095 .after_implement()
23096 .expect_err("no candidate declared anything; this is an ordinary failure");
23097
23098 assert!(
23099 err.to_string().contains("no candidate produced a change"),
23100 "{err}"
23101 );
23102 assert_eq!(runner.state.status, RunStatus::Failed);
23103 }
23104}
23105
23106#[derive(Debug)]
23109struct GraftRestoreFailed(String);
23110
23111impl std::fmt::Display for GraftRestoreFailed {
23112 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23113 write!(
23114 f,
23115 "could not restore the winner to its pre-graft commit: {}",
23116 self.0
23117 )
23118 }
23119}
23120
23121impl std::error::Error for GraftRestoreFailed {}
23122
23123#[cfg(test)]
23127mod resource_wait_tests {
23128 use super::*;
23129 use std::collections::BTreeMap;
23130
23131 fn sh(id: &str, script: &str) -> AgentSpec {
23132 AgentSpec {
23133 id: id.to_owned(),
23134 kind: crate::config::AgentKind::Command,
23135 model: None,
23136 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23137 extra_args: Vec::new(),
23138 env: BTreeMap::new(),
23139 prompt_delivery: None,
23140 max_turn_secs: None,
23141 }
23142 }
23143
23144 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23145 SeatJob {
23146 seat: SeatState::new("impl-A", &spec.id, 1),
23147 spec,
23148 cwd: dir.to_path_buf(),
23149 prompt: "do it".to_owned(),
23150 timeout,
23151 allow_write: true,
23152 sessions: false,
23153 artifacts: dir.join("art"),
23154 stem: "impl-A".to_owned(),
23155 handover: None,
23156 }
23157 }
23158
23159 fn state(resource_wait_secs: u64) -> RunState {
23160 let mut config = Config::default();
23161 config.graph.timeout_resource_wait = resource_wait_secs;
23162 RunState::new(
23163 PathBuf::from("/repo"),
23164 "main".to_owned(),
23165 "abc1234".to_owned(),
23166 "task".to_owned(),
23167 config,
23168 )
23169 }
23170
23171 fn hold(cache: &Path) -> crate::cache::Guard {
23172 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23173 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23174 crate::cache::AcquireOutcome::Acquired(g) => g,
23175 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23176 }
23177 }
23178
23179 async fn run(
23180 jobs: Vec<SeatJob>,
23181 sem: Arc<Semaphore>,
23182 cache: Option<&Path>,
23183 st: &mut RunState,
23184 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23185 let prompts = Prompts::default();
23186 let ctx = WaveCtx {
23187 run: "r",
23188 node: "implement",
23189 prompts: &prompts,
23190 cache,
23191 round: None,
23192 carry_seats: false,
23193 pauses: None,
23194 };
23195 wave(jobs, sem, &ctx, st, 0).await
23196 }
23197
23198 #[tokio::test]
23199 async fn verification_cache_wait_preserves_the_command_budget() {
23200 crate::run::pin_test_home();
23201 let tmp = tempfile::tempdir().unwrap();
23202 let cache = tmp.path().join("target");
23203 let guard = hold(&cache);
23204 let release = tokio::spawn(async move {
23205 tokio::time::sleep(Duration::from_millis(100)).await;
23206 guard.release();
23207 });
23208 let mut st = state(10);
23209 let budget = Duration::from_millis(50);
23210 let wait = repository_wait(&st, None);
23211 let (outcomes, _) = with_cache_lease(
23212 &mut st,
23213 Some(&cache),
23214 "gate",
23215 "gate",
23216 tmp.path(),
23217 "abc1234",
23218 budget,
23219 "test gate",
23220 &wait,
23221 |_, available| async move {
23222 assert_eq!(
23223 available, budget,
23224 "contention must not shorten verification"
23225 );
23226 (Vec::new(), false, Vec::new())
23227 },
23228 )
23229 .await
23230 .unwrap();
23231 release.await.unwrap();
23232 assert!(outcomes.is_empty(), "{outcomes:?}");
23233 assert!(
23234 st.events
23235 .iter()
23236 .any(|event| event.message.contains("waiting for"))
23237 );
23238 assert!(st.resource_wait.is_none());
23239 hold(&cache).release();
23241 }
23242
23243 #[tokio::test]
23244 async fn verification_cache_wait_expires_without_starting_commands() {
23245 crate::run::pin_test_home();
23246 let tmp = tempfile::tempdir().unwrap();
23247 let cache = tmp.path().join("target");
23248 let _held = hold(&cache);
23249 let mut st = state(0);
23250 let wait = repository_wait(&st, None);
23251 let (outcomes, _) = with_cache_lease(
23252 &mut st,
23253 Some(&cache),
23254 "gate",
23255 "gate",
23256 tmp.path(),
23257 "abc1234",
23258 Duration::from_secs(60),
23259 "test gate",
23260 &wait,
23261 |_, _| async { panic!("commands must not start without the cache lease") },
23262 )
23263 .await
23264 .unwrap();
23265 assert_eq!(outcomes.len(), 1);
23266 assert!(outcomes[0].resource_blocked);
23267 assert!(st.resource_wait.is_none());
23268 assert!(matches!(
23269 st.resource_block.unwrap().cause,
23270 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23271 ));
23272 }
23273
23274 #[tokio::test]
23275 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23276 crate::run::pin_test_home();
23277 let tmp = tempfile::tempdir().unwrap();
23278 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23279 let cache = tmp.path().join("cache");
23280 let mut state = state(21600);
23281 let wait = repository_wait(&state, None);
23282 let (outcomes, _) = with_cache_lease(
23283 &mut state,
23284 Some(&cache),
23285 "gate",
23286 "gate",
23287 tmp.path(),
23288 "head",
23289 Duration::from_secs(60),
23290 "gate",
23291 &wait,
23292 |_, _| async { panic!("freshness failure must not start verification") },
23293 )
23294 .await
23295 .unwrap();
23296 assert_eq!(outcomes.len(), 1);
23297 assert!(outcomes[0].resource_blocked);
23298 let block = state.resource_block.unwrap();
23299 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23300 assert!(!block.description().contains("timed out"));
23301 }
23302
23303 #[tokio::test]
23304 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23305 crate::run::pin_test_home();
23306 let tmp = tempfile::tempdir().unwrap();
23307 let cache = tmp.path().join("cache");
23308 let _held = hold(&cache);
23309 let mut state = state(21600);
23310 let pause = Pause::new();
23311 let interrupt = Pause::new();
23312 let cause = ParkCause::Operator;
23313 pause.park_for(cause.clone());
23314 let prompts = Prompts::default();
23315 let ctx = WaveCtx {
23316 run: "test",
23317 node: "fix",
23318 prompts: &prompts,
23319 cache: Some(&cache),
23320 round: None,
23321 carry_seats: false,
23322 pauses: Some((&pause, &interrupt)),
23323 };
23324 let result = wave(
23325 vec![job(
23326 sh("writer", "exit 99"),
23327 tmp.path(),
23328 Duration::from_secs(1),
23329 )],
23330 Arc::new(Semaphore::new(1)),
23331 &ctx,
23332 &mut state,
23333 0,
23334 )
23335 .await;
23336 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23337 assert_eq!(result[0].1.turns, 0);
23338 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23339 assert_eq!(
23340 state.resource_block.unwrap().cause,
23341 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23342 );
23343 assert!(state.active.is_empty());
23344 }
23345
23346 #[tokio::test]
23347 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23348 crate::run::pin_test_home();
23349 let tmp = tempfile::tempdir().unwrap();
23350 let cache = tmp.path().join("target-a");
23351 let guard = hold(&cache);
23352 tokio::spawn(async move {
23354 tokio::time::sleep(Duration::from_millis(1500)).await;
23355 guard.release();
23356 });
23357 let mut st = state(60);
23358 let results = run(
23359 vec![job(
23360 sh("alpha", "echo ok"),
23361 tmp.path(),
23362 Duration::from_secs(1),
23363 )],
23364 Arc::new(Semaphore::new(1)),
23365 Some(&cache),
23366 &mut st,
23367 )
23368 .await;
23369 assert!(
23370 matches!(results[0].2, AgentOutcome::Ok(_)),
23371 "the wait must not be charged to the agent"
23372 );
23373 assert!(st.resource_block.is_none());
23374 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23375 assert!(st.resource_wait.is_none() && st.active.is_empty());
23376 }
23377
23378 #[tokio::test]
23379 #[cfg(unix)]
23380 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23381 use std::os::fd::AsRawFd as _;
23382 crate::run::pin_test_home();
23383 let tmp = tempfile::tempdir().unwrap();
23384 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23385 let common = tmp.path().join(".git");
23386 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23387 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23388 assert_eq!(
23389 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23390 0
23391 );
23392 let mut st = state(1);
23393 st.config.coordination = crate::config::test_coordination();
23394 let mut reader = job(
23395 sh("reader", "echo must-not-run"),
23396 tmp.path(),
23397 Duration::from_secs(30),
23398 );
23399 reader.allow_write = false;
23400 let results = run(
23401 vec![reader.clone()],
23402 Arc::new(Semaphore::new(1)),
23403 None,
23404 &mut st,
23405 )
23406 .await;
23407 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23408 assert_eq!(results[0].1.turns, 0);
23409 assert!(FailClass::of(&results[0].2).is_none());
23410 assert_eq!(
23411 st.resource_block.as_ref().unwrap().kind,
23412 "repository coordination"
23413 );
23414 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23415 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23416 drop(external);
23417 let editor = crate::repo_lock::SeatLease::for_checkout(
23418 tmp.path(),
23419 true,
23420 crate::config::test_coordination().as_ref(),
23421 )
23422 .await
23423 .unwrap();
23424 let pause = Pause::new();
23425 let interrupt = Pause::new();
23426 let prompts = Prompts::default();
23427 let ctx = WaveCtx {
23428 run: "r",
23429 node: "review",
23430 prompts: &prompts,
23431 cache: None,
23432 round: None,
23433 carry_seats: false,
23434 pauses: Some((&pause, &interrupt)),
23435 };
23436 let (results, ()) = tokio::join!(
23437 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23438 async {
23439 tokio::time::sleep(Duration::from_millis(50)).await;
23440 pause.park();
23441 }
23442 );
23443 assert!(
23444 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23445 );
23446 assert_eq!(results[0].1.turns, 0);
23447 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23448 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23449 drop(editor);
23450 }
23451
23452 #[tokio::test]
23453 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23454 crate::run::pin_test_home();
23455 let tmp = tempfile::tempdir().unwrap();
23456 let cache = tmp.path().join("target-b");
23457 let _held = hold(&cache);
23458 let mut st = state(1);
23459 let results = run(
23460 vec![job(
23461 sh("alpha", "echo ok"),
23462 tmp.path(),
23463 Duration::from_secs(1),
23464 )],
23465 Arc::new(Semaphore::new(1)),
23466 Some(&cache),
23467 &mut st,
23468 )
23469 .await;
23470 let (_, seat, out) = &results[0];
23471 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23472 assert!(wave_blocked(&results));
23473 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23474 assert_eq!(seat.turns, 0, "nothing was asked");
23475 assert!(
23476 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23477 "no agent was spawned"
23478 );
23479 assert!(FailClass::of(out).is_none(), "not a failure class");
23480 let block = st.resource_block.as_ref().expect("recorded for the resume");
23481 assert_eq!(block.kind, "shared build cache");
23482 assert!(matches!(
23483 block.cause,
23484 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23485 ));
23486 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23487 assert!(st.active.is_empty());
23488 }
23489
23490 #[tokio::test]
23491 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23492 crate::run::pin_test_home();
23493 let tmp = tempfile::tempdir().unwrap();
23494 let mut st = state(1);
23495 let results = run(
23496 vec![job(
23497 sh("alpha", "echo ok"),
23498 tmp.path(),
23499 Duration::from_secs(1),
23500 )],
23501 Arc::new(Semaphore::new(0)),
23502 None,
23503 &mut st,
23504 )
23505 .await;
23506 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23507 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23508 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23509 }
23510
23511 #[tokio::test]
23512 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23513 crate::run::pin_test_home();
23514 let tmp = tempfile::tempdir().unwrap();
23515 let mut st = state(60);
23516 let results = run(
23517 vec![job(
23518 sh("alpha", "sleep 30"),
23519 tmp.path(),
23520 Duration::from_secs(1),
23521 )],
23522 Arc::new(Semaphore::new(1)),
23523 None,
23524 &mut st,
23525 )
23526 .await;
23527 let out = &results[0].2;
23528 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23529 let class = FailClass::of(out).expect("a failure");
23530 assert_eq!(class, FailClass::Timeout);
23531 assert!(should_hand_over(None, &class), "existing handover policy");
23532 assert!(st.resource_block.is_none());
23533 }
23534}