1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 if state.startup.as_ref().is_some_and(|s| !s.complete)
1470 || (state.status == RunStatus::Prep
1471 && state.instruction.trim().is_empty()
1472 && state.candidates.is_empty()
1473 && state.prep_complete == Some(false))
1474 {
1475 bail!(
1476 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477 );
1478 }
1479 crate::pr_head::ensure_owner(&state)?;
1480 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481 bail!(
1482 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483 );
1484 }
1485 if let Some(to) = &state.released_to {
1486 bail!(
1487 "run {} cannot be resumed: its worktree was released to run {}",
1488 state.short(),
1489 crate::run::short_of(to)
1490 );
1491 }
1492 let roles = state.config.resolve_roles()?;
1493 if let Some(id) = &state.config.graph.selected_implementer
1494 && state.candidates.iter().any(|c| c.agent != *id)
1495 {
1496 bail!(
1497 "replacement implementer pin does not match the saved candidates; task must stay held"
1498 );
1499 }
1500 let max_parallel = state.config.graph.max_parallel.max(1);
1501 Ok(Self {
1502 state,
1503 roles,
1504 sem: Arc::new(Semaphore::new(max_parallel)),
1505 pause: Pause::new(),
1506 interrupt: Pause::new(),
1507 })
1508 }
1509
1510 pub async fn execute(&mut self) -> Result<()> {
1517 anyhow::ensure!(
1518 self.state.startup.as_ref().is_none_or(|s| s.complete),
1519 "startup incomplete; inspect retained evidence before explicit recovery"
1520 );
1521 if let Some(selection) = &self.state.pr_recovery
1522 && (selection.source_proof.is_some() || selection.source_run.is_some())
1523 {
1524 Box::pin(selection.verify_source(&self.state.repo)).await?;
1525 }
1526 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529 || self.state.land_approval.is_some()
1530 || self.state.merge_decision.is_some()
1531 || !self.state.merge_repairs.is_empty()
1532 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533 {
1534 let claim = FixClaim::acquire(&self.state.dir())?;
1535 let disk = RunState::load(&self.state.id)?;
1536 let checkpoint = |s: &RunState| {
1537 (
1538 s.merge_repairs.len(),
1539 s.merge_repairs
1540 .last()
1541 .map(|r| (r.question.clone(), r.phase)),
1542 s.land_approval
1543 .as_ref()
1544 .map(|a| (a.question.clone(), a.head.clone())),
1545 )
1546 };
1547 if checkpoint(&disk) != checkpoint(&self.state) {
1548 bail!(
1549 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550 );
1551 }
1552 Some(claim)
1553 } else {
1554 None
1555 };
1556 let mut execution = self.claim_driver().await?;
1557 let runner = &mut *execution.runner;
1558 let wait = runner.repository_wait();
1559 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562 let result = match result {
1563 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564 record_repository_block(&mut runner.state, &e);
1565 runner.park_here()?;
1566 Ok(())
1567 }
1568 other => other,
1569 };
1570 execution.driver.finish(&mut runner.state).await?;
1571 let ended = if result.is_err() {
1572 Some({
1573 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574 if let Err(error) = &result {
1575 notice.message = format!(
1576 "Run {} stopped with an error: {}",
1577 runner.state.id,
1578 error.to_string().lines().next().unwrap_or("unknown error")
1579 );
1580 }
1581 notice
1582 })
1583 } else {
1584 crate::notices::run_ended(&runner.state)
1585 };
1586 if let Some(notice) = ended {
1587 crate::notices::raise(notice);
1588 }
1589 result
1590 }
1591
1592 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594 }
1595
1596 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597 let wait = self.repository_wait();
1598 coordinate_operation(&mut self.state, cwd, &wait).await
1599 }
1600
1601 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603 let owner = match crate::pr_head::owner_lock_wait(
1604 &crate::run::home(),
1605 &self.state.id,
1606 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607 )
1608 .await
1609 {
1610 Ok(owner) => owner,
1611 Err(error) => {
1612 let detail = format!("worktree maintenance or ownership update: {error:#}");
1613 self.state.resource_block = Some(crate::run::ResourceBlock {
1617 cause: crate::run::ResourceBlockCause::Other,
1618 node: "start".into(),
1619 kind: "worktree maintenance".into(),
1620 detail: detail.clone(),
1621 at: Timestamp::now(),
1622 });
1623 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624 }
1625 };
1626 let disk = RunState::load(&self.state.id)?;
1627 crate::pr_head::ensure_owner(&disk)?;
1628 let claimed =
1629 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630 anyhow::ensure!(
1635 disk.liveness(claimed) != crate::run::Liveness::Live,
1636 "run already has a live driver; wait for its execution to end"
1637 );
1638 if let Some(to) = &disk.released_to {
1639 bail!(
1640 "run {} cannot continue: its worktree was released to run {}",
1641 self.state.short(),
1642 crate::run::short_of(to)
1643 );
1644 }
1645 anyhow::ensure!(
1646 serde_json::to_value(&disk.base_sync_recovery)?
1647 == serde_json::to_value(&self.state.base_sync_recovery)?,
1648 "base recovery changed since this run was loaded; reload the saved run"
1649 );
1650 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651 let mut execution = DriverExecution {
1652 runner: self,
1653 driver,
1654 };
1655 let _owner = owner;
1658 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659 .await?;
1660 let state = &mut execution.runner.state;
1661 state.parked = false;
1666 state.resource_wait = None;
1667 state.resource_block = None;
1668 state.clear_active();
1675 let pid = std::process::id();
1690 state.driver_pid = Some(pid);
1691 state.driver_started_at = crate::proc::process_started_at(pid);
1692 state.driver_exited = false;
1693 state.driver_claimed = Some(true);
1694 state.begin_pass();
1695 execution.driver.bind(state);
1696 state.save()?;
1697 execution.driver.record(state)?;
1698 Ok(execution)
1699 }
1700
1701 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1704 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1705 if let Err(error) = &result
1706 && !error.is::<crate::repo_lock::ResourceBlocked>()
1707 && self
1708 .state
1709 .repair_grants
1710 .last()
1711 .is_some_and(|g| g.conflict.is_none())
1712 {
1713 self.state.repair_grants.last_mut().unwrap().phase =
1714 crate::repair_budget::Phase::Complete;
1715 self.state.status = RunStatus::Blocked;
1716 self.state.save()?;
1717 bail!(
1718 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1719 self.state.passes.last().map_or(0, |p| p.index)
1720 );
1721 }
1722 result
1723 }
1724
1725 async fn execute_granted_repair(&mut self) -> Result<()> {
1728 use crate::repair_budget::{Kind, Phase};
1729 let index = self.state.repair_grants.len() - 1;
1730 let winner = self
1731 .state
1732 .winner()
1733 .cloned()
1734 .context("repair winner missing")?;
1735 loop {
1736 let grant = self.state.repair_grants[index].clone();
1737 match grant.phase {
1738 Phase::Pending => {
1739 if grant.spent >= grant.receipt.request.rounds {
1740 self.state.repair_grants[index].phase = Phase::Complete;
1741 self.state.status = RunStatus::Blocked;
1742 self.state.save()?;
1743 return Ok(());
1744 }
1745 self.state.repair_grants[index].spent += 1;
1748 self.state.repair_grants[index].phase = Phase::Fixing;
1749 self.state.save()?;
1750 match grant.receipt.request.kind {
1751 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1752 Kind::Gate => {
1753 let failed = self.state.gate.clone();
1754 let _ = self.gate_fix_round(&winner, &failed).await?;
1755 }
1756 Kind::Review => {
1757 let mut record = self
1758 .state
1759 .reviews
1760 .last()
1761 .cloned()
1762 .context("review failure missing")?;
1763 let round = record.round;
1764 let step = self.run_fix_round(&winner, &mut record).await?;
1765 self.state.reviews[round - 1] = record;
1766 if matches!(step, FixStep::Recorded { report_lost: true }) {
1767 self.state.repair_grants[index].phase = Phase::Complete;
1768 self.state.status = RunStatus::Blocked;
1769 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1770 self.state.save()?;
1771 return Ok(());
1772 }
1773 if matches!(step, FixStep::Resource) {
1774 self.stop_for_resource()?;
1775 return Ok(());
1776 }
1777 self.state.save()?;
1778 }
1779 }
1780 if self.resource_halt()? {
1781 return Ok(());
1782 }
1783 }
1784 Phase::Fixing => {
1785 if grant.receipt.request.kind == Kind::BaseConflict {
1786 self.repair_base_conflict(false).await?;
1787 }
1788 let _coordination = self.coordinate(&winner.worktree).await?;
1791 let rescued = git::rescue_commit(
1792 &winner.worktree,
1793 "magi: preserve interrupted granted repair",
1794 )
1795 .await?;
1796 self.state.note_withheld("repair-grant", &rescued.withheld);
1797 self.state
1800 .verification_history
1801 .push(crate::run::VerificationEvidence {
1802 superseded_by: None,
1803 archived_at: Some(Timestamp::now()),
1804 fixes_spent: Some(self.state.gate_fixes.len()),
1805 gate_head: self.state.gate_head.take(),
1806 gate_ran: std::mem::take(&mut self.state.gate_ran),
1807 gate: std::mem::take(&mut self.state.gate),
1808 e2e: self.state.head_e2e.take(),
1809 });
1810 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1811 self.state
1812 .verification_history
1813 .push(crate::run::VerificationEvidence {
1814 superseded_by: None,
1815 archived_at: Some(Timestamp::now()),
1816 fixes_spent: Some(self.state.gate_fixes.len()),
1817 gate_head: None,
1818 gate_ran: false,
1819 gate: Vec::new(),
1820 e2e: Some(e2e),
1821 });
1822 }
1823 self.state.repair_grants[index].review_start = self.state.reviews.len();
1824 self.state.repair_grants[index].head =
1825 git::rev_parse(&winner.worktree, "HEAD").await?;
1826 self.state.repair_grants[index].phase = Phase::Reviewing;
1827 if grant.receipt.request.kind == Kind::BaseConflict {
1828 self.state.base_sync_recovery.stop = None;
1829 if let Some(sync) = &mut self.state.base_sync {
1830 sync.tip = grant
1831 .receipt
1832 .request
1833 .tip
1834 .clone()
1835 .context("repair base tip missing")?;
1836 sync.conflict = None;
1837 sync.behind = 0;
1838 }
1839 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1840 }
1841 self.state.save()?;
1842 }
1843 Phase::Reviewing => {
1844 self.review_loop_pass().await?;
1845 if self.resource_halt()? {
1846 return Ok(());
1847 }
1848 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1849 let clean = self.state.reviews.len() > grant.review_start
1850 && self.state.reviews.last().is_some_and(|r| {
1851 r.head == head
1852 && r.clean
1853 && r.expected > 0
1854 && r.answered == r.expected
1855 && matches!(
1856 r.e2e_status(),
1857 E2eStatus::Passed | E2eStatus::NotConfigured
1858 )
1859 });
1860 if !clean {
1861 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1863 == Kind::Review
1864 && grant.spent < grant.receipt.request.rounds
1865 && self.state.reviews.last().is_some_and(|r| {
1866 r.fix.is_none()
1867 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1868 }) {
1869 Phase::Pending
1870 } else {
1871 Phase::Complete
1872 };
1873 self.state.status = RunStatus::Blocked;
1874 self.state.event(
1875 "repair-grant",
1876 "fresh review did not clear the repaired head; no publication",
1877 );
1878 self.state.save()?;
1879 if self.state.repair_grants[index].phase == Phase::Pending {
1880 continue;
1881 }
1882 return Ok(());
1883 }
1884 self.state.repair_grants[index].phase = Phase::Gating;
1885 self.state.save()?;
1886 }
1887 Phase::Gating => {
1888 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1889 self.state.reconcile_verification_head(&head);
1890 let wait = self.repository_wait();
1891 if self
1892 .state
1893 .reviews
1894 .last()
1895 .is_none_or(|r| r.head != head || !r.clean)
1896 {
1897 self.state.status = RunStatus::Blocked;
1898 self.state.repair_grants[index].phase = Phase::Complete;
1899 self.state.event(
1900 "repair-grant",
1901 "head changed since fresh review; repair held",
1902 );
1903 self.state.save()?;
1904 return Ok(());
1905 }
1906 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1907 return Ok(());
1908 }
1909 let outcomes = self.run_gate(&winner).await?;
1910 if verify_inconclusive(&outcomes) {
1911 return Ok(());
1912 }
1913 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1914 if after != head {
1915 self.state.repair_grants[index].review_start = self.state.reviews.len();
1916 self.state.repair_grants[index].head = after;
1917 self.state.repair_grants[index].phase = Phase::Reviewing;
1918 self.state.status = RunStatus::Blocked;
1919 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1920 self.state.save()?;
1921 return Ok(());
1922 }
1923 let passed = outcomes.iter().all(CommandOutcome::ok);
1924 if passed
1927 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1928 {
1929 return Ok(());
1930 }
1931 self.state.repair_grants[index].succeeded = passed;
1932 self.state.gate = outcomes;
1933 self.state.gate_ran = true;
1934 self.state.status = if passed {
1935 RunStatus::Gating
1936 } else {
1937 RunStatus::Blocked
1938 };
1939 self.state.repair_grants[index].phase = if !passed
1940 && grant.receipt.request.kind == Kind::Gate
1941 && grant.spent < grant.receipt.request.rounds
1942 {
1943 Phase::Pending
1944 } else {
1945 Phase::Complete
1946 };
1947 if passed && self.state.config.merge.mode == MergeMode::Pr {
1948 self.state.pr_publication_pending = true;
1950 } else if passed {
1951 self.state.status = RunStatus::Ready;
1952 }
1953 self.state.save()?;
1954 if !passed {
1955 continue;
1956 }
1957 if self.state.config.merge.mode != MergeMode::Pr {
1961 self.state.status = RunStatus::Ready;
1962 self.state.event(
1963 "repair-grant",
1964 "repair verified locally; grant carries no merge authorization",
1965 );
1966 self.state.save()?;
1967 return Ok(());
1968 }
1969 return self.merge().await;
1970 }
1971 Phase::Complete => return Ok(()),
1972 }
1973 }
1974 }
1975
1976 async fn execute_graph(&mut self) -> Result<()> {
1977 let archive_home = crate::run::home().canonicalize()?;
1981 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1982 let repo = self.state.repo.clone();
1983 let _coordination = self.coordinate(&repo).await?;
1984 ensure_setup_config(&mut self.state, &repo).await?;
1985 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1986 }
1987 self.state.configure_recovery_resume()?;
1988 if self
1989 .state
1990 .repair_grants
1991 .last()
1992 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1993 {
1994 return self.execute_granted_repair().await;
1995 }
1996 if self
1997 .state
1998 .repair_grants
1999 .last()
2000 .is_some_and(|g| !g.succeeded)
2001 {
2002 self.state.status = RunStatus::Blocked;
2003 self.state.save()?;
2004 return Ok(());
2005 }
2006 self.persist_candidates().await?;
2009 if self.state.base_sync_blocked() {
2012 self.sync_to_base().await?;
2013 if !self.state.base_sync_blocked()
2014 && self.state.status != RunStatus::AlreadyInBase
2015 && !self.resource_halt()?
2016 {
2017 self.finish_after_tally().await?;
2018 }
2019 return Ok(());
2020 }
2021 if self
2022 .state
2023 .merge_decision
2024 .as_ref()
2025 .is_some_and(|decision| !decision.closed)
2026 && self.state.status != RunStatus::Landing
2027 {
2028 if crate::merge_decision::resumable_hold(&self.state) {
2029 self.state.merge = Some(MergeOutcome {
2030 mode: MergeMode::Pr,
2031 ok: false,
2032 detail: self
2033 .state
2034 .merge_decision
2035 .as_ref()
2036 .unwrap()
2037 .target
2038 .pr
2039 .clone(),
2040 empty: false,
2041 });
2042 self.run_land().await?;
2043 self.settle_questions();
2044 return Ok(());
2045 }
2046 bail!(
2047 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2048 );
2049 }
2050 if self.state.config.graph.selected_implementer.is_some()
2051 && self.state.status == RunStatus::Blocked
2052 && self.state.tally.is_none()
2053 && self.state.candidate_preservation.is_empty()
2054 && self.state.interrupted_implementations.is_empty()
2055 {
2056 bail!(
2057 "selected implementation is held; explicitly release the owning task to retry it"
2058 );
2059 }
2060
2061 if self.state.merge_repairs.last().is_some_and(|r| {
2062 matches!(
2063 r.phase,
2064 crate::run::MergeRepairPhase::Requested
2065 | crate::run::MergeRepairPhase::Fixing
2066 | crate::run::MergeRepairPhase::Reviewing
2067 )
2068 }) {
2069 if self.repair_merge_review().await? {
2070 self.run_land().await?;
2071 }
2072 self.settle_questions();
2073 return Ok(());
2074 }
2075 if self.state.pr_publication_pending {
2078 self.merge().await?;
2079 if self.resource_halt()? {
2080 return Ok(());
2081 }
2082 self.state.save()?;
2083 self.settle_questions();
2084 return Ok(());
2085 }
2086 if self.state.status == RunStatus::Stalled {
2099 if self.recover_stall().await? {
2100 self.finish_after_tally().await?;
2101 } else {
2102 self.state.save()?;
2104 }
2105 return Ok(());
2106 }
2107 if self.state.status == RunStatus::Landing
2117 || crate::handover::landing_checkpoint(&self.state)
2118 {
2119 self.state.land_sync_pending = true;
2120 self.state.save()?;
2121 self.run_land().await?;
2122 self.settle_questions();
2127 return Ok(());
2128 }
2129 if self.state.github_text.is_some()
2134 && self.state.merge.is_none()
2135 && self.state.status == RunStatus::Gating
2136 {
2137 self.merge().await?;
2138 if self.resource_halt()? {
2139 return Ok(());
2140 }
2141 self.state.save()?;
2142 return Ok(());
2143 }
2144 if let Err(e) = self.prep().await {
2145 if e.is::<crate::repo_lock::ResourceBlocked>() {
2146 return Err(e);
2147 }
2148 self.state.status = RunStatus::Blocked;
2149 self.state
2150 .prep_failure
2151 .get_or_insert_with(|| format!("{e:#}"));
2152 self.state
2153 .event("prep", format!("preparation stopped: {e:#}"));
2154 self.state.save()?;
2155 return Err(e);
2156 }
2157 if self.park_here()? {
2158 return Ok(());
2159 }
2160 self.advise().await?;
2161 if self.park_here()? {
2162 return Ok(());
2163 }
2164 self.implement().await?;
2165 if self.park_here()? {
2166 return Ok(());
2167 }
2168 if self.state.status == RunStatus::VerifiedNoop {
2172 return Ok(());
2173 }
2174 self.judge().await?;
2175 if self.park_here()? {
2176 return Ok(());
2177 }
2178 self.deliberate().await?;
2179 if self.park_here()? {
2180 return Ok(());
2181 }
2182 self.vote().await?;
2183 if self.park_here()? {
2184 return Ok(());
2185 }
2186 self.tally()?;
2187 if self.state.status == RunStatus::Stalled {
2192 self.state.save()?;
2196 return Ok(());
2197 }
2198 self.finish_after_tally().await?;
2199 Ok(())
2200 }
2201
2202 fn stop_for_resource(&mut self) -> Result<()> {
2207 if self.pause.parked() || self.interrupt.parked() {
2210 self.park_here()?;
2211 return Ok(());
2212 }
2213 if self.state.resource_block.is_some() {
2214 self.state.status = RunStatus::Blocked;
2215 }
2216 self.state.save()
2217 }
2218
2219 fn resource_halt(&mut self) -> Result<bool> {
2221 if self.state.resource_block.is_none() {
2222 return Ok(false);
2223 }
2224 self.stop_for_resource()?;
2225 Ok(true)
2226 }
2227
2228 fn park_here(&mut self) -> Result<bool> {
2235 let parking = self.pause.parked() || self.interrupt.parked();
2236 if !parking && self.state.resource_block.is_some() {
2237 self.stop_for_resource()?;
2238 return Ok(true);
2239 }
2240 if !self.pause.parked() && !self.interrupt.parked() {
2245 return Ok(false);
2246 }
2247 if self.state.parked {
2248 return Ok(true);
2249 }
2250 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2251 if let Some(block) = &mut self.state.resource_block
2252 && matches!(
2253 block.cause,
2254 crate::run::ResourceBlockCause::Cancelled { .. }
2255 )
2256 {
2257 block.cause = crate::run::ResourceBlockCause::Cancelled {
2258 park: cause.clone(),
2259 };
2260 }
2261 let why = match cause.as_ref().map(ParkCause::label) {
2262 Some(reason) => format!(
2263 "parked after `{}` ({reason}) — resume to carry on from here",
2264 self.state.status.as_str()
2265 ),
2266 None => format!(
2267 "parked after `{}` — resume to carry on from here",
2268 self.state.status.as_str()
2269 ),
2270 };
2271 self.state.event("park", why);
2272 self.state
2273 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2274 self.state.parked = true;
2275 self.state.save()?;
2276 Ok(true)
2277 }
2278
2279 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2281 if self.state.prep_failure.is_none() {
2282 return Ok(());
2283 }
2284 let failed_at = self
2285 .state
2286 .events
2287 .iter()
2288 .rev()
2289 .find(|event| event.node == "prep")
2290 .map_or(self.state.updated_at, |event| event.at);
2291 anyhow::ensure!(
2292 approval_at > failed_at,
2293 "preparation retry requires a new explicit release after the failure"
2294 );
2295 self.state.prep_failure = None;
2296 self.state.prep_complete = Some(false);
2297 self.state
2298 .event("prep-retry", "operator approved retry of preparation");
2299 Ok(())
2300 }
2301
2302 pub fn on_pause(&mut self, pause: Pause) {
2304 self.pause = pause;
2305 }
2306
2307 pub fn watch_interrupt(&mut self, pause: Pause) {
2313 self.interrupt = pause;
2314 }
2315
2316 fn settle_questions(&mut self) {
2336 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2337 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2338 }
2339 }
2340
2341 async fn finish_after_tally(&mut self) -> Result<()> {
2344 self.graft().await?;
2347 if self.resource_halt()? {
2348 return Ok(());
2349 }
2350 self.fold_losers().await?;
2351 self.sync_to_base().await?;
2356 if self.hold_base_sync().await? || self.resource_halt()? {
2357 return Ok(());
2358 }
2359 if self.state.status == RunStatus::AlreadyInBase {
2360 return Ok(());
2361 }
2362 self.review_loop().await?;
2363 if self.resource_halt()? {
2364 return Ok(());
2365 }
2366 self.sync_to_base().await?;
2367 if self.hold_base_sync().await? || self.resource_halt()? {
2368 return Ok(());
2369 }
2370 if self.state.status == RunStatus::AlreadyInBase {
2371 return Ok(());
2372 }
2373 self.gate().await?;
2374 self.merge().await?;
2375 if self.resource_halt()? {
2376 return Ok(());
2377 }
2378 self.state.save()?;
2379 Ok(())
2380 }
2381
2382 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2389 let result = match ensure_setup_config(&mut self.state, repo).await {
2390 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2391 Err(e) => Err(e),
2392 };
2393 if let Err(e) = result {
2394 git::worktree_remove_clean(repo, wt).await.ok();
2395 if in_prep {
2396 self.rollback_prep(repo).await;
2397 }
2398 self.state.status = RunStatus::Blocked;
2399 self.state.event(
2400 "setup",
2401 format!("worktree setup failed in {}: {e:#}", wt.display()),
2402 );
2403 self.state.save()?;
2404 return Err(e);
2405 }
2406 Ok(())
2407 }
2408
2409 async fn rollback_prep(&mut self, repo: &Path) {
2411 let root = self.state.worktree_root();
2412 for c in std::mem::take(&mut self.state.candidates) {
2413 if matches!(
2414 git::worktree_remove_clean(repo, &c.worktree).await,
2415 Ok(true)
2416 ) {
2417 git::branch_delete(repo, &c.branch).await.ok();
2418 } else {
2419 self.state.candidates.push(c);
2420 }
2421 }
2422 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2423 for j in 1..=seats {
2424 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2425 .await
2426 .ok();
2427 }
2428 for k in 1..=self.state.config.graph.advisors {
2429 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2430 .await
2431 .ok();
2432 }
2433 }
2434
2435 async fn prep(&mut self) -> Result<()> {
2436 if let Some(reason) = &self.state.prep_failure {
2437 bail!(
2438 "candidate preparation previously failed: {reason}; correct the source before retrying"
2439 );
2440 }
2441 if self
2442 .state
2443 .prep_complete
2444 .unwrap_or(!self.state.candidates.is_empty())
2445 {
2446 return Ok(());
2447 }
2448 let repo = self.state.repo.clone();
2449 let _coordination = self.coordinate(&repo).await?;
2452 let base = self.state.base_commit.clone();
2453 let plan = self.state.sources.plan(&repo, &base).await?;
2454 self.rollback_prep(&repo).await;
2455 if !self.state.candidates.is_empty() {
2456 bail!(
2457 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2458 );
2459 }
2460 self.state.prep_complete = Some(false);
2461 self.state.status = RunStatus::Prep;
2462 self.state.save()?;
2463 let repo = self.state.repo.clone();
2464 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2465 let root = self.state.worktree_root();
2466 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2467
2468 let hooks_dir = self.state.dir().join("hooks");
2471 if self.state.config.blind.commit_msg_hook {
2472 std::fs::create_dir_all(&hooks_dir)
2473 .with_context(|| format!("create {}", hooks_dir.display()))?;
2474 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2475 let path = hooks_dir.join("commit-msg");
2476 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2477 make_executable(&path)?;
2478 git::acquire_worktree_config(&repo).await?;
2486 self.state.enabled_worktree_config = true;
2487 }
2488
2489 ensure_setup_config(&mut self.state, &repo).await?;
2492
2493 for (index, (spec, label)) in self
2494 .roles
2495 .implementers
2496 .clone()
2497 .into_iter()
2498 .zip(labels)
2499 .enumerate()
2500 {
2501 let branch = self.state.branch_for(label);
2502 let worktree = root.join(format!("cand-{label}"));
2503 self.state.event(
2507 "prep-intent",
2508 serde_json::json!({
2509 "branch": branch,
2510 "worktree": worktree,
2511 "start": start,
2512 "picks": plan.picks,
2513 })
2514 .to_string(),
2515 );
2516 self.state.save()?;
2517 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2518 if self.state.config.blind.commit_msg_hook {
2519 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2520 }
2521 git::local_exclude(&worktree, "/.magi/").await?;
2522 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2523 git::worktree_remove_clean(&repo, &worktree).await.ok();
2526 git::branch_delete(&repo, &branch).await.ok();
2527 self.rollback_prep(&repo).await;
2528 self.state.status = RunStatus::Blocked;
2529 self.state
2530 .event("prep", format!("worktree setup failed: {e:#}"));
2531 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2532 self.state.save()?;
2533 return Err(e);
2534 }
2535 self.state.candidates.push(Candidate {
2536 index,
2537 label,
2538 agent: spec.id.clone(),
2539 branch,
2540 worktree,
2541 summary: String::new(),
2542 stat: String::new(),
2543 files: 0,
2544 commits: 0,
2545 empty: false,
2546 failed: None,
2547 verified_noop: None,
2548 duration_ms: 0,
2549 folded: false,
2550 });
2551 self.state.save()?;
2552 for pick in &plan.picks {
2553 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2554 {
2555 self.state.status = RunStatus::Blocked;
2556 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2557 self.state
2558 .event("prep", format!("cannot apply referenced commit: {e}"));
2559 self.state.save()?;
2560 return Err(e);
2561 }
2562 }
2563 }
2564
2565 for j in 1..=self.roles.judges.len() {
2566 let wt = root.join(format!("judge-{j}"));
2567 if !wt.exists() {
2568 git::worktree_add_detached(&repo, &wt, &base).await?;
2569 self.setup_seat_worktree(&repo, &wt, true).await?;
2570 }
2571 }
2572
2573 if self.state.config.graph.advise {
2581 for k in 1..=self.state.config.graph.advisors {
2582 let wt = root.join(format!("advisor-{k}"));
2583 if !wt.exists() {
2584 git::worktree_add_detached(&repo, &wt, &base).await?;
2585 self.setup_seat_worktree(&repo, &wt, true).await?;
2586 }
2587 }
2588 }
2589
2590 let authors: Vec<&str> = self
2595 .roles
2596 .implementers
2597 .iter()
2598 .map(|a| a.id.as_str())
2599 .collect();
2600 let overlap: Vec<String> = self
2601 .roles
2602 .judges
2603 .iter()
2604 .enumerate()
2605 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2606 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2607 .collect();
2608 if !overlap.is_empty() {
2609 let note = format!(
2610 "{} also authored a candidate; blind, but the panel is less \
2611 independent than {} distinct agents would be",
2612 overlap.join(", "),
2613 self.roles.judges.len()
2614 );
2615 self.state.event("prep", note);
2616 }
2617
2618 self.state.event(
2619 "prep",
2620 format!(
2621 "{} candidates, {} judges, base {} ({})",
2622 self.state.candidates.len(),
2623 self.roles.judges.len(),
2624 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2625 self.state.base_branch
2626 ),
2627 );
2628 self.state.prep_complete = Some(true);
2629 self.state.status = RunStatus::Implementing;
2630 self.state.save()?;
2631 Ok(())
2632 }
2633
2634 async fn advise(&mut self) -> Result<()> {
2667 let implement_untouched = self
2668 .state
2669 .candidates
2670 .iter()
2671 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2672 if !self.state.config.graph.advise || self.state.advise_attempted {
2673 return Ok(());
2674 }
2675 if !implement_untouched {
2676 self.state.event(
2677 "advise",
2678 "skipping the design-deliberation stage: at least one \
2679 candidate already shows implementation progress, so this \
2680 run is past the point the stage exists to run before"
2681 .to_owned(),
2682 );
2683 self.state.advise_attempted = true;
2684 self.state.save()?;
2685 return Ok(());
2686 }
2687 let run_id = self.state.id.clone();
2688 let prompts = self.state.config.prompts.clone();
2689 let instruction = self.state.instruction.clone();
2690 let language = self.state.config.graph.language.clone();
2691 let root = self.state.worktree_root();
2692 let n = self.state.config.graph.advisors;
2693 let where_recorded = self.state.dir().join("run.json");
2694
2695 let seats = match self.state.config.advisors() {
2696 Ok(seats) if !seats.is_empty() => seats,
2697 Ok(_) => {
2698 self.state.event(
2699 "advise",
2700 format!(
2701 "[graph] advisors is 0; skipping the design-deliberation \
2702 stage and continuing without a synthesis brief (see {})",
2703 where_recorded.display()
2704 ),
2705 );
2706 self.state.advise_attempted = true;
2707 self.state.save()?;
2708 return Ok(());
2709 }
2710 Err(e) => {
2711 self.state.event(
2712 "advise",
2713 format!(
2714 "could not resolve advisor seats ({e:#}); continuing \
2715 without a design-deliberation brief (see {})",
2716 where_recorded.display()
2717 ),
2718 );
2719 self.state.advise_attempted = true;
2720 self.state.save()?;
2721 return Ok(());
2722 }
2723 };
2724
2725 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2726 let artifacts = agent::artifacts_dir(&self.state.dir());
2727 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2728
2729 let mut jobs = Vec::new();
2730 for (i, spec) in seats.iter().cloned().enumerate() {
2731 let seat_key = format!("advisor-{}", i + 1);
2732 let seat = self.seat(&seat_key, &spec.id);
2733 jobs.push(SeatJob {
2734 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2735 spec,
2736 seat,
2737 cwd: worktrees[i % worktrees.len()].clone(),
2738 timeout,
2739 allow_write: false,
2740 sessions: false,
2741 artifacts: artifacts.clone(),
2742 stem: seat_key,
2743 handover: None,
2744 });
2745 }
2746
2747 self.state.event(
2748 "advise",
2749 format!(
2750 "{} advisor seat(s) sketching a design in parallel",
2751 jobs.len()
2752 ),
2753 );
2754 let mut quota_losses = Vec::new();
2755 let cache = self.state.config.cache_dir();
2756 let ctx = WaveCtx {
2757 carry_seats: false,
2758 pauses: Some((&self.pause, &self.interrupt)),
2759 run: &run_id,
2760 node: "advise",
2761 prompts: &prompts,
2762 cache: cache.as_deref(),
2763 round: None,
2764 };
2765 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2766 let results = ask_json_wave::<Proposal>(
2767 jobs,
2768 Arc::clone(&self.sem),
2769 self.state.config.graph.retries,
2770 &advisor_roster,
2771 &ctx,
2772 &mut quota_losses,
2773 &mut self.state,
2774 &|p: &Proposal| p.validate(),
2775 )
2776 .await;
2777 if self.state.resource_block.is_some() {
2778 return self.stop_for_resource();
2779 }
2780 self.state.quota.extend(quota_losses);
2781
2782 let mut records = Vec::with_capacity(results.len());
2783 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2784 let agent_id = seat.agent.clone();
2785 self.state.seats.insert(seat.key.clone(), seat);
2786 match res {
2787 Ok((proposal, out)) => {
2788 self.state
2789 .event("advise", format!("advisor-{} proposed a design", i + 1));
2790 records.push(advise::AdvisorRecord::proposed(
2791 i + 1,
2792 agent_id,
2793 proposal,
2794 out.duration_ms,
2795 ));
2796 }
2797 Err(e) => {
2798 self.state.event(
2799 "advise",
2800 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2801 );
2802 records.push(advise::AdvisorRecord::failed(
2803 i + 1,
2804 agent_id,
2805 e.to_string(),
2806 ));
2807 }
2808 }
2809 }
2810
2811 let mut advice = advise::Advice {
2812 records,
2813 synthesis: None,
2814 };
2815 if advice.proposals().is_empty() {
2816 self.state.event(
2817 "advise",
2818 "no advisor produced a usable proposal; continuing without a \
2819 synthesis brief"
2820 .to_owned(),
2821 );
2822 } else {
2823 match self
2824 .synthesize_brief(
2825 &advice,
2826 &instruction,
2827 &language,
2828 &worktrees[0],
2829 &artifacts,
2830 &run_id,
2831 &prompts,
2832 cache.as_deref(),
2833 )
2834 .await
2835 {
2836 Ok(Some(text)) => {
2837 self.state.event(
2838 "advise",
2839 "synthesized a design brief for the implementer".to_owned(),
2840 );
2841 advice.synthesis = Some(text);
2842 }
2843 Ok(None) => {
2844 self.state.event(
2845 "advise",
2846 "the synthesis seat produced nothing usable; continuing \
2847 without a design brief"
2848 .to_owned(),
2849 );
2850 }
2851 Err(e) => {
2852 if e.is::<crate::repo_lock::ResourceBlocked>() {
2853 return Err(e);
2854 }
2855 self.state.event(
2856 "advise",
2857 format!("could not synthesize a design brief: {e:#}"),
2858 );
2859 }
2860 }
2861 }
2862 advise::apply_reflection(&mut advice);
2863
2864 self.state.advice = Some(advice);
2865 self.state.advise_attempted = true;
2866 self.state.save()?;
2867 Ok(())
2868 }
2869
2870 #[allow(clippy::too_many_arguments)]
2882 async fn synthesize_brief(
2883 &mut self,
2884 advice: &advise::Advice,
2885 instruction: &str,
2886 language: &str,
2887 cwd: &Path,
2888 artifacts: &Path,
2889 run_id: &str,
2890 prompts: &Prompts,
2891 cache: Option<&Path>,
2892 ) -> Result<Option<String>> {
2893 let chain = agent::pick_chain(
2894 &self.state.config.agents,
2895 self.state.config.roles.synthesizer.as_ref(),
2896 &agent::installed,
2897 "synthesizer",
2898 )?;
2899 let proposals = advice.proposals();
2900 let mut prompt = prompt::with_overlay(
2901 prompt::synthesize_brief(instruction, &proposals, language),
2902 prompts.overlay("advise"),
2903 );
2904 if cache.is_some() {
2905 prompt.push('\n');
2910 prompt.push_str(&prompt::build_cache_note("advise", false));
2911 }
2912 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2913 let mut last = None;
2918 for (n, spec) in chain.iter().enumerate() {
2919 if n > 0 {
2920 self.state
2921 .event("advise", format!("synthesis falling back to {}", spec.id));
2922 }
2923 let mut seat = self.seat("advise-synthesis", &spec.id);
2924 let wait = self.repository_wait();
2925 let outcome = wait
2926 .scope(agent::invoke(
2927 spec,
2928 &mut seat,
2929 &Invocation {
2930 coordination: self.state.config.coordination.as_ref(),
2931 fetched_base: None,
2932 cwd,
2933 prompt: &prompt,
2934 timeout,
2935 allow_write: false,
2936 sessions: false,
2937 artifacts,
2938 stem: &if n == 0 {
2939 "advise-synthesis".to_owned()
2940 } else {
2941 format!("advise-synthesis-{}", spec.id)
2942 },
2943 run: run_id,
2944 node: "advise",
2945 cache_dir: None,
2946 attachments: &[],
2947 writable: &[],
2948 },
2949 ))
2950 .await;
2951 if outcome.is_ok() {
2952 self.state.seats.insert(seat.key.clone(), seat);
2953 }
2954 let advance = agent::chain_advances(&outcome);
2955 last = Some(outcome);
2956 if !advance {
2957 break;
2958 }
2959 }
2960 let out = last.expect("a chain holds at least one agent")?;
2963 if !out.usable() {
2964 return Ok(None);
2965 }
2966 let text =
2967 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2968 Ok((!text.trim().is_empty()).then_some(text))
2969 }
2970
2971 async fn implement(&mut self) -> Result<()> {
2974 self.persist_candidates().await?;
2975 let run_id = self.state.id.clone();
2980 let prompts = self.state.config.prompts.clone();
2981 let todo: Vec<usize> = self
2982 .state
2983 .candidates
2984 .iter()
2985 .enumerate()
2986 .filter(|(_, c)| {
2987 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2988 && c.failed.is_none()
2989 && !c.empty
2990 })
2991 .map(|(i, _)| i)
2992 .collect();
2993 if todo.is_empty() {
2994 return self.after_implement();
2995 }
2996 self.state.status = RunStatus::Implementing;
2997
2998 let language = self.state.config.graph.language.clone();
2999 let github_language = self.state.config.graph.github_language.clone();
3000 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3001 let sessions = self.state.config.graph.sessions;
3002 let artifacts = agent::artifacts_dir(&self.state.dir());
3003 let brief = self
3007 .state
3008 .advice
3009 .as_ref()
3010 .and_then(|a| a.synthesis.as_deref())
3011 .map(str::to_owned);
3012 let attachments = self.state.attachments.clone();
3013
3014 let mut jobs = Vec::new();
3015 for &i in &todo {
3016 let (index, label, worktree) = {
3017 let c = &self.state.candidates[i];
3018 (c.index, c.label, c.worktree.clone())
3019 };
3020 let spec = self.roles.implementers[index].clone();
3021 let seat_key = format!("impl-{label}");
3022 let seat = self.seat(&seat_key, &spec.id);
3023 let instruction = seeded_instruction(&self.state);
3024 jobs.push(SeatJob {
3025 spec,
3026 seat,
3027 prompt: prompt::implement_in(
3028 &instruction,
3029 &worktree.to_string_lossy(),
3030 &language,
3031 &github_language,
3032 brief.as_deref(),
3033 &attachments,
3034 ),
3035 cwd: worktree,
3036 timeout,
3037 allow_write: true,
3038 sessions,
3039 artifacts: artifacts.clone(),
3040 stem: format!("impl-{label}"),
3041 handover: None,
3042 });
3043 }
3044
3045 self.state.event(
3046 "implement",
3047 format!("{} candidates in parallel", jobs.len()),
3048 );
3049 let mut sent = jobs.clone();
3055 let cache = self.state.config.cache_dir();
3056 let ctx = WaveCtx {
3057 carry_seats: false,
3058 pauses: Some((&self.pause, &self.interrupt)),
3059 run: &run_id,
3060 node: "implement",
3061 prompts: &prompts,
3062 cache: cache.as_deref(),
3063 round: None,
3064 };
3065 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3066 if !wave_blocked(&results) {
3067 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3068 .await;
3069 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3070 .await?;
3071 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3072 .await;
3073 }
3074 if !results.is_empty()
3077 && results
3078 .iter()
3079 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3080 {
3081 let losses: Vec<_> = results
3082 .iter()
3083 .map(|(_, seat, out)| {
3084 let AgentOutcome::Quota(o) = out else {
3085 unreachable!()
3086 };
3087 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3088 })
3089 .collect();
3090 self.state.quota.extend(losses.clone());
3091 for (_, seat, _) in results {
3092 self.state.seats.insert(seat.key.clone(), seat);
3093 }
3094 for &i in &todo {
3095 let label = self.state.candidates[i].label;
3096 self.state.interrupted_implementations.insert(label);
3097 self.state
3098 .candidate_preservation
3099 .insert(label, "machine outage awaiting persistence".into());
3100 }
3101 block_machine_outage(&mut self.state, "implement", &losses);
3104 self.state.save()?;
3105 self.persist_candidates().await?;
3106 for &i in &todo {
3107 self.state
3108 .interrupted_implementations
3109 .insert(self.state.candidates[i].label);
3110 }
3111 return self.stop_for_resource();
3112 }
3113 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3114
3115 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3116 && results
3117 .iter()
3118 .any(|(_, _, out)| FailClass::of(out).is_some());
3119 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3120 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3121 if self
3125 .state
3126 .candidate_preservation
3127 .contains_key(&self.state.candidates[i].label)
3128 {
3129 self.state.seats.insert(seat.key.clone(), seat);
3130 }
3131 continue;
3132 }
3133 let seat_key = seat.key.clone();
3134 let agent = seat.agent.clone();
3143 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3144 self.state.seats.insert(seat.key.clone(), seat);
3145 let label = self.state.candidates[i].label;
3146 self.state.interrupted_implementations.remove(&label);
3147
3148 let (summary, duration, failed, verified_claim) = match out {
3149 AgentOutcome::Ok(o) => {
3150 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3151 let failed = (!o.usable()).then(|| {
3152 if o.timed_out {
3153 "agent timed out".to_owned()
3154 } else {
3155 format!(
3156 "agent exited with {}",
3157 crate::notices::exit_status(o.exit_code)
3158 )
3159 }
3160 });
3161 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3162 (text, o.duration_ms, failed, verified_claim)
3163 }
3164 AgentOutcome::Dropped(o) => {
3170 let why = o
3171 .dropped
3172 .as_ref()
3173 .map(|d| d.why.as_str())
3174 .unwrap_or("the CLI ended the stream without delivering its answer");
3175 (
3176 String::new(),
3177 o.duration_ms,
3178 Some(format!("the CLI dropped the stream ({why})")),
3179 None,
3180 )
3181 }
3182 AgentOutcome::Quota(o) => {
3183 self.state.quota.push(QuotaLoss::from_report(
3184 seat_key,
3185 "implement",
3186 o.quota.as_ref(),
3187 ));
3188 (
3189 String::new(),
3190 o.duration_ms,
3191 Some("rate limited (quota); produced no change".to_owned()),
3192 None,
3193 )
3194 }
3195 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3196 (String::new(), 0, Some(e), None)
3197 }
3198 };
3199
3200 let c = &mut self.state.candidates[i];
3203 if !exhausted_the_fallback_chain {
3204 c.agent = agent;
3205 }
3206 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3207 c.duration_ms = duration;
3208 c.failed = failed;
3209 c.verified_noop = verified_claim;
3210 self.state
3211 .candidate_preservation
3212 .insert(label, "awaiting persistence".to_owned());
3213 }
3214 self.state.save()?;
3215 self.persist_candidates().await?;
3216
3217 if resource_blocked {
3218 return self.stop_for_resource();
3219 }
3220 if selected_failed {
3221 self.state.status = RunStatus::Blocked;
3222 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3223 self.state.save()?;
3224 bail!(
3225 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3226 );
3227 }
3228 self.after_implement()
3229 }
3230
3231 async fn persist_candidates(&mut self) -> Result<()> {
3233 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3234 for label in labels {
3235 let i = self
3236 .state
3237 .candidates
3238 .iter()
3239 .position(|c| c.label == label)
3240 .context("pending candidate is missing")?;
3241 let c = self.state.candidates[i].clone();
3242 let interrupted = self.state.interrupted_implementations.contains(&label);
3243 let result = async {
3244 let wait = crate::repo_lock::WaitPolicy::new(
3245 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3246 || false,
3247 );
3248 let _coordination =
3249 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3250 if let Err(e) = git::snapshot_uncommitted(
3253 &c.worktree,
3254 &self
3255 .state
3256 .dir()
3257 .join("artifacts")
3258 .join(format!("candidate-{label}-recovery")),
3259 )
3260 .await
3261 {
3262 self.state.event(
3263 "preserve",
3264 format!("candidate {label} snapshot incomplete: {e:#}"),
3265 );
3266 }
3267 let rescued = git::rescue_commit(
3268 &c.worktree,
3269 &format!("magi: candidate {label} (uncommitted work)"),
3270 )
3271 .await?;
3272 self.state.note_withheld("implement", &rescued.withheld);
3273 let base = &self.state.base_commit;
3274 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3275 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3276 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3277 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3278 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3279 if c.empty && commits > 0 && !patch.trim().is_empty()
3280 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3281 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3282 || self.state.tally.is_some())
3283 {
3284 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3288 }
3289 let c = &mut self.state.candidates[i];
3290 c.commits = commits;
3291 c.stat = stat;
3292 c.files = files;
3293 let no_changes = commits == 0 || patch.trim().is_empty();
3294 c.empty = no_changes && !interrupted;
3295 if interrupted && no_changes {
3296 c.failed = None;
3297 c.verified_noop = None;
3298 }
3299 if !c.empty {
3300 c.failed = None;
3301 c.verified_noop = None;
3302 }
3303 self.state.event(
3304 "preserve",
3305 format!("candidate {label}: {files} files, {commits} commits"),
3306 );
3307 Ok::<_, anyhow::Error>(())
3308 }
3309 .await;
3310 match result {
3311 Ok(()) => {
3312 self.state.candidate_preservation.remove(&label);
3313 if !interrupted || self.state.candidates[i].files > 0 {
3314 self.state.interrupted_implementations.remove(&label);
3315 }
3316 }
3317 Err(e) => {
3318 let reason = format!("candidate {label} preservation failed: {e:#}");
3319 self.state.event("preserve", &reason);
3320 self.state.candidate_preservation.insert(label, reason);
3321 }
3322 }
3323 if self.state.candidate_preservation.is_empty()
3324 && self.state.status == RunStatus::Blocked
3325 && self.state.candidates.iter().all(|c| c.failed.is_none())
3326 {
3327 self.state.status = RunStatus::Implementing;
3331 self.state.event(
3332 "preserve",
3333 "candidate persistence recovered; implementation can continue",
3334 );
3335 }
3336 self.state.save()?;
3337 }
3338 if !self.state.candidate_preservation.is_empty() {
3339 self.state.status = RunStatus::Blocked;
3340 self.state.save()?;
3341 bail!(
3342 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3343 );
3344 }
3345 Ok(())
3346 }
3347
3348 async fn resume_undelivered(
3376 &mut self,
3377 results: &mut [(usize, SeatState, AgentOutcome)],
3378 sent: &[SeatJob],
3379 prompts: &Prompts,
3380 run_id: &str,
3381 ) {
3382 for (wi, seat, out) in results.iter_mut() {
3383 let Some(dropped) = (match &*out {
3384 AgentOutcome::Dropped(o) => o.dropped.clone(),
3385 _ => None,
3386 }) else {
3387 continue;
3388 };
3389 let Some(job) = sent.get(*wi) else { continue };
3390 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3392 self.state.event(
3393 "implement",
3394 format!(
3395 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3396 work is in the tree",
3397 seat.key, dropped.output_tokens, dropped.why
3398 ),
3399 );
3400 continue;
3401 }
3402 if !has_context(&job.spec, seat, job.sessions) {
3410 self.state.event(
3411 "implement",
3412 format!(
3413 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3414 is no session left to resume",
3415 seat.key, dropped.output_tokens, dropped.why
3416 ),
3417 );
3418 continue;
3419 }
3420 self.state.event(
3421 "implement",
3422 format!(
3423 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3424 conversation",
3425 seat.key, dropped.output_tokens, dropped.why
3426 ),
3427 );
3428 let mut retry = job.clone();
3429 retry.seat = seat.clone();
3430 retry.prompt = prompt::resume_after_drop(&dropped.why);
3431 retry.timeout = retry_budget(job.timeout, true);
3432 retry.stem = format!("{}-resume", job.stem);
3433 let cache = self.state.config.cache_dir();
3434 let ctx = WaveCtx {
3435 carry_seats: false,
3436 pauses: Some((&self.pause, &self.interrupt)),
3437 run: run_id,
3438 node: "implement",
3439 prompts,
3440 cache: cache.as_deref(),
3441 round: None,
3442 };
3443 let (resumed_seat, resumed) =
3444 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3445 *seat = resumed_seat;
3446 *out = resumed;
3447 }
3448 }
3449
3450 async fn resume_seat_handovers(
3515 &mut self,
3516 results: &mut [(usize, SeatState, AgentOutcome)],
3517 sent: &mut [SeatJob],
3518 prompts: &Prompts,
3519 run_id: &str,
3520 ) -> Result<()> {
3521 let instruction = seeded_instruction(&self.state);
3522 let language = self.state.config.graph.language.clone();
3523 let github_language = self.state.config.graph.github_language.clone();
3524 let brief = self
3525 .state
3526 .advice
3527 .as_ref()
3528 .and_then(|a| a.synthesis.as_deref())
3529 .map(str::to_owned);
3530 let attachments = self.state.attachments.clone();
3531 for (wi, seat, out) in results.iter_mut() {
3532 let others: BTreeSet<String> = sent
3535 .iter()
3536 .enumerate()
3537 .filter(|(j, _)| j != wi)
3538 .map(|(_, j)| j.spec.id.clone())
3539 .collect();
3540 let Some(job) = sent.get_mut(*wi) else {
3541 continue;
3542 };
3543 let start = self
3548 .roles
3549 .implementer_roster
3550 .iter()
3551 .position(|s| s.id == job.spec.id)
3552 .unwrap_or(0);
3553 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3554 let mut fallback_attempt = 0usize;
3555 let mut prev: Option<FailClass> = None;
3556 while let Some(cur) = FailClass::of(&*out) {
3557 if !should_hand_over(prev.as_ref(), &cur) {
3558 break;
3559 }
3560 let Some(next) =
3561 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3562 .cloned()
3563 else {
3564 break;
3565 };
3566 tried.insert(next.id.clone());
3567 fallback_attempt += 1;
3568
3569 let coordination = self.coordinate(&job.cwd).await?;
3570 if let Ok(r) = git::rescue_commit(
3571 &job.cwd,
3572 &format!(
3573 "magi: candidate {} (uncommitted work before {} fallback)",
3574 seat.key,
3575 if cur == FailClass::Quota {
3576 "quota"
3577 } else {
3578 "handover"
3579 }
3580 ),
3581 )
3582 .await
3583 {
3584 self.state.note_withheld("implement", &r.withheld);
3585 }
3586 drop(coordination);
3587
3588 record_handover(
3589 &mut self.state,
3590 "implement",
3591 &seat.key,
3592 &seat.agent,
3593 &next.id,
3594 &cur,
3595 &fail_reason(&*out),
3596 );
3597 prev = Some(cur.clone());
3598
3599 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3600 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3601 job.spec = next.clone();
3609 let mut retry = job.clone();
3610 retry.seat = new_seat;
3611 retry.prompt = prompt::implement_in(
3612 &instruction,
3613 &job.cwd.to_string_lossy(),
3614 &language,
3615 &github_language,
3616 brief.as_deref(),
3617 &attachments,
3618 );
3619 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3620 let cache = self.state.config.cache_dir();
3621 let ctx = WaveCtx {
3622 carry_seats: false,
3623 pauses: Some((&self.pause, &self.interrupt)),
3624 run: run_id,
3625 node: "implement",
3626 prompts,
3627 cache: cache.as_deref(),
3628 round: None,
3629 };
3630 let (fallback_seat, fallback_out) = run_one(
3631 retry,
3632 Arc::clone(&self.sem),
3633 &ctx,
3634 &mut self.state,
3635 fallback_attempt,
3636 )
3637 .await;
3638 *seat = fallback_seat;
3639 *out = fallback_out;
3640 }
3641 }
3642 Ok(())
3643 }
3644
3645 async fn resume_unconfirmed_commands(
3669 &mut self,
3670 results: &mut [(usize, SeatState, AgentOutcome)],
3671 sent: &[SeatJob],
3672 prompts: &Prompts,
3673 run_id: &str,
3674 ) {
3675 for (wi, seat, out) in results.iter_mut() {
3676 let AgentOutcome::Ok(o) = &*out else {
3677 continue;
3678 };
3679 if !has_unconfirmed_command(&o.commands) {
3680 continue;
3681 }
3682 let Some(job) = sent.get(*wi) else { continue };
3683 if !has_context(&job.spec, seat, job.sessions) {
3684 self.state.event(
3685 "implement",
3686 format!(
3687 "{}: the reply named a command whose own CLI never confirmed the exit \
3688 status of, but there is no session left to resume",
3689 seat.key
3690 ),
3691 );
3692 continue;
3693 }
3694 self.state.event(
3695 "implement",
3696 format!(
3697 "{}: the reply named a command whose own CLI never confirmed the exit \
3698 status of; resuming the conversation",
3699 seat.key
3700 ),
3701 );
3702 let mut retry = job.clone();
3703 retry.seat = seat.clone();
3704 retry.prompt = prompt::resume_incomplete(
3705 "a command in your last reply had no confirmed exit status",
3706 );
3707 retry.timeout = retry_budget(job.timeout, true);
3708 retry.stem = format!("{}-confirm", job.stem);
3709 let cache = self.state.config.cache_dir();
3710 let ctx = WaveCtx {
3711 carry_seats: false,
3712 pauses: Some((&self.pause, &self.interrupt)),
3713 run: run_id,
3714 node: "implement",
3715 prompts,
3716 cache: cache.as_deref(),
3717 round: None,
3718 };
3719 let (resumed_seat, resumed) =
3720 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3721 *seat = resumed_seat;
3722 *out = resumed;
3723 }
3724 }
3725
3726 async fn continue_fix_report(
3747 &mut self,
3748 mut seat: SeatState,
3749 parse_err: String,
3750 job: &SeatJob,
3751 prompts: &Prompts,
3752 run_id: &str,
3753 round: usize,
3754 ) -> (
3755 SeatState,
3756 Option<FixReport>,
3757 Option<String>,
3758 ContinuationRecord,
3759 ) {
3760 let mut last_err = parse_err;
3761 let mut cumulative_wait_ms = 0u64;
3762 let mut attempts = 0usize;
3763 loop {
3764 if !has_context(&job.spec, &seat, job.sessions) {
3765 self.state.event(
3766 "fix",
3767 format!(
3768 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3769 session left to resume into"
3770 ),
3771 );
3772 let outcome = if attempts == 0 {
3773 ContinuationOutcome::NoSession
3774 } else {
3775 ContinuationOutcome::Exhausted
3776 };
3777 return (
3778 seat,
3779 None,
3780 Some(format!("unparsable fix report: {last_err}")),
3781 ContinuationRecord {
3782 attempts,
3783 cumulative_wait_ms,
3784 outcome,
3785 },
3786 );
3787 }
3788 if attempts >= MAX_FIX_CONTINUATIONS {
3789 self.state.event(
3790 "fix",
3791 format!(
3792 "round {round}: fixer's reply still had no adoption report after \
3793 {attempts} continuation(s) ({last_err}); giving up"
3794 ),
3795 );
3796 return (
3797 seat,
3798 None,
3799 Some(format!(
3800 "unparsable fix report after {attempts} continuation(s): {last_err}"
3801 )),
3802 ContinuationRecord {
3803 attempts,
3804 cumulative_wait_ms,
3805 outcome: ContinuationOutcome::Exhausted,
3806 },
3807 );
3808 }
3809 attempts += 1;
3810 self.state.event(
3811 "fix",
3812 format!(
3813 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3814 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3815 ),
3816 );
3817 let mut retry = job.clone();
3818 retry.seat = seat.clone();
3819 retry.prompt = prompt::resume_incomplete(&last_err);
3820 retry.timeout = retry_budget(job.timeout, true);
3821 retry.stem = format!("{}-continue{attempts}", job.stem);
3822 let cache = self.state.config.cache_dir();
3823 let ctx = WaveCtx {
3824 carry_seats: false,
3825 pauses: Some((&self.pause, &self.interrupt)),
3826 run: run_id,
3827 node: "fix",
3828 prompts,
3829 cache: cache.as_deref(),
3830 round: Some(round),
3831 };
3832 let (resumed_seat, resumed_out) = run_one(
3833 retry,
3834 Arc::clone(&self.sem),
3835 &ctx,
3836 &mut self.state,
3837 attempts,
3838 )
3839 .await;
3840 seat = resumed_seat;
3841 match resumed_out {
3842 AgentOutcome::Ok(o) => {
3843 cumulative_wait_ms += o.duration_ms;
3844 match verdict::extract_json::<FixReport>(&o.text) {
3845 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3846 self.state.event(
3847 "fix",
3848 format!(
3849 "round {round}: fixer's adoption report recovered after \
3850 {attempts} continuation(s)"
3851 ),
3852 );
3853 return (
3854 seat,
3855 Some(report),
3856 None,
3857 ContinuationRecord {
3858 attempts,
3859 cumulative_wait_ms,
3860 outcome: ContinuationOutcome::Resumed,
3861 },
3862 );
3863 }
3864 Ok(_) => {
3872 last_err = "the reply parsed, but it reported a command whose own CLI \
3873 never confirmed an exit status"
3874 .to_owned();
3875 }
3876 Err(e) => last_err = e.to_string(),
3877 }
3878 }
3879 AgentOutcome::Quota(o) => {
3880 cumulative_wait_ms += o.duration_ms;
3881 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3882 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3883 self.state.quota.push(loss);
3884 self.state.event(
3885 "fix",
3886 format!(
3887 "round {round}: continuation rate limited (quota); not retrying now"
3888 ),
3889 );
3890 return (
3891 seat,
3892 None,
3893 Some("rate limited (quota) while recovering the fix report".to_owned()),
3894 ContinuationRecord {
3895 attempts,
3896 cumulative_wait_ms,
3897 outcome: ContinuationOutcome::QuotaLost,
3898 },
3899 );
3900 }
3901 AgentOutcome::Dropped(o) => {
3902 cumulative_wait_ms += o.duration_ms;
3903 let why = o
3904 .dropped
3905 .as_ref()
3906 .map(|d| d.why.as_str())
3907 .unwrap_or("the CLI ended the stream without delivering its answer");
3908 last_err = format!("the CLI dropped the stream ({why})");
3909 }
3910 AgentOutcome::Failed(e) => last_err = e,
3911 AgentOutcome::ResourceBlocked(e) => {
3914 return (
3915 seat,
3916 None,
3917 Some(format!("the continuation was not started: {e}")),
3918 ContinuationRecord {
3919 attempts: attempts.saturating_sub(1),
3920 cumulative_wait_ms,
3921 outcome: ContinuationOutcome::NoSession,
3922 },
3923 );
3924 }
3925 }
3926 }
3927 }
3928
3929 fn after_implement(&mut self) -> Result<()> {
3930 if !self.state.candidate_preservation.is_empty() {
3931 bail!("candidate preservation incomplete; judging is unsafe");
3932 }
3933 if self.state.leaks.is_empty() {
3935 let cfg = self.state.config.blind.clone();
3936 let mut leaks = Vec::new();
3937 for c in &self.state.candidates {
3938 let Some(patch) =
3939 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3940 else {
3941 continue;
3942 };
3943 leaks.extend(blind::scan(
3944 &format!("candidate {} patch", c.label),
3945 &patch,
3946 &cfg.vendor_tokens,
3947 ));
3948 }
3949 if !leaks.is_empty() {
3950 let summary = leaks
3951 .iter()
3952 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3953 .collect::<Vec<_>>()
3954 .join(", ");
3955 match cfg.on_leak {
3956 LeakPolicy::Fail => {
3957 self.state.status = RunStatus::Failed;
3958 self.state
3959 .event("blind", format!("vendor text in a patch: {summary}"));
3960 self.state.leaks = leaks;
3961 self.state.save()?;
3962 self.settle_questions();
3963 bail!(
3964 "blind.on_leak = \"fail\" and vendor text reached a \
3965 judged patch: {summary}"
3966 );
3967 }
3968 LeakPolicy::Redact => self.state.event(
3969 "blind",
3970 format!("redacting vendor text for judging: {summary}"),
3971 ),
3972 LeakPolicy::Warn => self.state.event(
3973 "blind",
3974 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3975 ),
3976 }
3977 self.state.leaks = leaks;
3978 }
3979 }
3980
3981 if self.state.viable().is_empty() {
3982 if self.state.all_candidates_verified_noop() {
3983 self.state.status = RunStatus::VerifiedNoop;
3994 self.state.save()?;
3995 self.settle_questions();
3996 return Ok(());
3997 }
3998 self.state.status = RunStatus::Failed;
3999 self.state.save()?;
4000 self.settle_questions();
4001 bail!("no candidate produced a change; nothing to judge");
4002 }
4003 self.state.status = RunStatus::Judging;
4004 self.state.save()?;
4005 Ok(())
4006 }
4007
4008 async fn judge(&mut self) -> Result<()> {
4011 let run_id = self.state.id.clone();
4016 let prompts = self.state.config.prompts.clone();
4017 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4018 return Ok(());
4019 }
4020 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4021 if viable.len() == 1 {
4022 self.state.judge_skipped = true;
4029 self.state.event(
4030 "judge",
4031 format!(
4032 "only candidate {} produced a change; judging skipped",
4033 viable[0].label
4034 ),
4035 );
4036 self.state.save()?;
4037 return Ok(());
4038 }
4039 self.state.status = RunStatus::Judging;
4040
4041 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4042 let language = self.state.config.graph.language.clone();
4043 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4044 let sessions = self.state.config.graph.sessions;
4045 let artifacts = agent::artifacts_dir(&self.state.dir());
4046 let root = self.state.worktree_root();
4047 let base_short = short(&self.state.base_commit);
4048
4049 let mut jobs = Vec::new();
4050 let mut orders = Vec::new();
4051 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4052 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4053 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4054 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4055 let seat_key = format!("judge-{}", j + 1);
4056 let seat = self.seat(&seat_key, &spec.id);
4057 jobs.push(SeatJob {
4058 prompt: prompt::judge(
4059 &self.state.instruction,
4060 &views,
4061 self.roles.judges.len(),
4062 &base_short,
4063 &language,
4064 ),
4065 spec,
4066 seat,
4067 cwd: root.join(format!("judge-{}", j + 1)),
4068 timeout,
4069 allow_write: false,
4070 sessions,
4071 artifacts: artifacts.clone(),
4072 stem: format!("judge-{}", j + 1),
4073 handover: None,
4074 });
4075 }
4076
4077 self.state.event(
4078 "judge",
4079 format!(
4080 "{} judges ranking {} candidates blind",
4081 jobs.len(),
4082 viable.len()
4083 ),
4084 );
4085 let labels_for_check = labels.clone();
4086 let mut quota_losses = Vec::new();
4087 let cache = self.state.config.cache_dir();
4088 let ctx = WaveCtx {
4089 carry_seats: false,
4090 pauses: Some((&self.pause, &self.interrupt)),
4091 run: &run_id,
4092 node: "judge",
4093 prompts: &prompts,
4094 cache: cache.as_deref(),
4095 round: None,
4096 };
4097 let results = ask_json_wave::<Ranking>(
4098 jobs,
4099 Arc::clone(&self.sem),
4100 self.state.config.graph.retries,
4101 &self.roles.judge_roster,
4102 &ctx,
4103 &mut quota_losses,
4104 &mut self.state,
4105 &move |r: &Ranking| r.validate(&labels_for_check),
4106 )
4107 .await;
4108 if self.state.resource_block.is_some() {
4109 return self.stop_for_resource();
4110 }
4111 self.state.quota.extend(quota_losses);
4112
4113 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4114 let agent_id = seat.agent.clone();
4115 self.state.seats.insert(seat.key.clone(), seat);
4116 let mut record = Judgement {
4117 judge: j + 1,
4118 seat: format!("judge-{}", j + 1),
4119 agent: agent_id,
4120 ranking: Vec::new(),
4121 reasons: BTreeMap::new(),
4122 confidence: None,
4123 strengths: BTreeMap::new(),
4124 order: orders[j].clone(),
4125 failed: None,
4126 duration_ms: 0,
4127 };
4128 match res {
4129 Ok((ranking, out)) => {
4130 record.ranking = ranking.normalized();
4131 record.reasons = ranking.reasons;
4132 record.strengths = ranking.strengths;
4133 record.confidence = ranking.confidence;
4134 record.duration_ms = out.duration_ms;
4135 self.state.event(
4136 "judge",
4137 format!(
4138 "judge {} ranked {}",
4139 j + 1,
4140 record.ranking.iter().collect::<String>()
4141 ),
4142 );
4143 }
4144 Err(e) => {
4145 record.failed = Some(e.to_string());
4146 self.state
4147 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4148 }
4149 }
4150 self.state.judgements.push(record);
4151 self.state.save()?;
4152 }
4153 Ok(())
4154 }
4155
4156 async fn deliberate(&mut self) -> Result<()> {
4159 if self.hold_base_sync().await? {
4160 return Ok(());
4161 }
4162 let run_id = self.state.id.clone();
4167 let prompts = self.state.config.prompts.clone();
4168 let rounds = self.state.config.graph.deliberate_rounds;
4169 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4171 return Ok(());
4172 }
4173 let tops: Vec<char> = self
4174 .state
4175 .judgements
4176 .iter()
4177 .filter_map(|j| j.ranking.first().copied())
4178 .collect();
4179 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4180 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4181 self.state.event(
4182 "deliberate",
4183 format!("judges agreed on {} outright; no deliberation", tops[0]),
4184 );
4185 }
4186 self.state.status = RunStatus::Voting;
4187 self.state.save()?;
4188 return Ok(());
4189 }
4190
4191 self.state.status = RunStatus::Deliberating;
4192 self.state.event(
4193 "deliberate",
4194 format!(
4195 "split: first choices were {} — opening {rounds} round(s)",
4196 tops.iter().collect::<String>()
4197 ),
4198 );
4199
4200 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4201 let language = self.state.config.graph.language.clone();
4202 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4203 let sessions = self.state.config.graph.sessions;
4204 let artifacts = agent::artifacts_dir(&self.state.dir());
4205 let root = self.state.worktree_root();
4206 let base_short = short(&self.state.base_commit);
4207
4208 for round in self.state.deliberation.len() + 1..=rounds {
4212 let mut turns: Vec<DeliberationTurn> = Vec::new();
4213 let mut round_losses = Vec::new();
4214 let mut asked = 0;
4215 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4216 if self.state.judgements[j].failed.is_some() {
4217 continue;
4218 }
4219 let seat_key = format!("judge-{}", j + 1);
4220 let spec = self.occupant(&seat_key, spec);
4221 let mut seat = self.seat(&seat_key, &spec.id);
4222 let transcript = self.transcript(&turns, j);
4223 let build = |context: Option<&str>| {
4224 prompt::deliberate(
4225 &self.state.instruction,
4226 context,
4227 &transcript,
4228 round,
4229 rounds,
4230 &language,
4231 )
4232 };
4233 let block = self.candidate_block(&viable, &base_short);
4234 let full = build(Some(&block));
4235 let text = if has_context(&spec, &seat, sessions) {
4236 build(None)
4237 } else {
4238 full.clone()
4239 };
4240 let job = SeatJob {
4241 spec,
4242 seat: seat.clone(),
4243 prompt: text,
4244 cwd: root.join(format!("judge-{}", j + 1)),
4245 timeout,
4246 allow_write: false,
4247 sessions,
4248 artifacts: artifacts.clone(),
4249 stem: format!("delib-{round}-judge-{}", j + 1),
4250 handover: Some(full),
4251 };
4252 let cache = self.state.config.cache_dir();
4253 let ctx = WaveCtx {
4254 carry_seats: false,
4255 pauses: Some((&self.pause, &self.interrupt)),
4256 run: &run_id,
4257 node: "deliberate",
4258 prompts: &prompts,
4259 cache: cache.as_deref(),
4260 round: None,
4261 };
4262 let mut losses = Vec::new();
4266 asked += 1;
4267 let mut results = ask_wave_with::<String>(
4268 vec![job],
4269 Arc::clone(&self.sem),
4270 0,
4271 &self.roles.judge_roster,
4272 &ctx,
4273 &mut losses,
4274 &mut self.state,
4275 &|text: &str| {
4276 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4277 },
4278 )
4279 .await;
4280 if self.state.resource_block.is_some() {
4281 return self.stop_for_resource();
4282 }
4283 round_losses.extend(losses.iter().cloned());
4284 self.state.quota.extend(losses);
4285 let (updated, res, _) = results.pop().expect("one job in, one result out");
4286 seat = updated;
4287 let agent_id = seat.agent.clone();
4288 self.state.seats.insert(seat.key.clone(), seat);
4289 let body = match res {
4290 Ok((body, _)) => body,
4291 Err(e) => {
4294 self.state
4295 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4296 continue;
4297 }
4298 };
4299 let tentative = verdict::extract_json::<Position>(&body)
4300 .ok()
4301 .and_then(|p| p.tentative)
4302 .and_then(|s| s.trim().chars().next())
4303 .map(|c| c.to_ascii_uppercase());
4304 self.state.event(
4305 "deliberate",
4306 format!(
4307 "round {round}: judge {} now favours {}",
4308 j + 1,
4309 tentative.map_or("—".to_owned(), |c| c.to_string())
4310 ),
4311 );
4312 turns.push(DeliberationTurn {
4313 judge: j + 1,
4314 agent: agent_id,
4315 body: blind::sanitize_prose(&body, &self.state.config.blind),
4316 tentative,
4317 });
4318 }
4319 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4320 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4321 return self.stop_for_resource();
4322 }
4323 self.state
4324 .deliberation
4325 .push(DeliberationRound { round, turns });
4326 self.state.save()?;
4327 }
4328
4329 self.state.status = RunStatus::Voting;
4330 self.state.save()?;
4331 Ok(())
4332 }
4333
4334 async fn vote(&mut self) -> Result<()> {
4337 if self.hold_base_sync().await? {
4338 return Ok(());
4339 }
4340 let run_id = self.state.id.clone();
4345 let prompts = self.state.config.prompts.clone();
4346 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4347 return Ok(());
4348 }
4349 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4350 if viable.len() == 1 {
4351 return Ok(());
4352 }
4353 self.state.status = RunStatus::Voting;
4354
4355 let language = self.state.config.graph.language.clone();
4356 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4357 let sessions = self.state.config.graph.sessions;
4358 let artifacts = agent::artifacts_dir(&self.state.dir());
4359 let root = self.state.worktree_root();
4360 let base_short = short(&self.state.base_commit);
4361 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4362
4363 let mut jobs = Vec::new();
4364 let mut seats_at = Vec::new();
4365 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4366 if self
4367 .state
4368 .judgements
4369 .get(j)
4370 .is_some_and(|r| r.failed.is_some())
4371 {
4372 continue;
4373 }
4374 let seat_key = format!("judge-{}", j + 1);
4375 let spec = self.occupant(&seat_key, spec);
4376 let seat = self.seat(&seat_key, &spec.id);
4377 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4378 let text = if has_context(&spec, &seat, sessions) {
4379 prompt::final_vote(&viable, &language)
4380 } else {
4381 full.clone()
4382 };
4383 jobs.push(SeatJob {
4384 spec,
4385 seat,
4386 prompt: text,
4387 cwd: root.join(format!("judge-{}", j + 1)),
4388 timeout,
4389 allow_write: false,
4390 sessions,
4391 artifacts: artifacts.clone(),
4392 stem: format!("vote-judge-{}", j + 1),
4393 handover: Some(full),
4394 });
4395 seats_at.push(j);
4396 }
4397
4398 self.state.event(
4399 "vote",
4400 format!(
4401 "collecting {} final votes one by one, privately",
4402 jobs.len()
4403 ),
4404 );
4405 let allowed = viable.clone();
4406 let mut quota_losses = Vec::new();
4407 let cache = self.state.config.cache_dir();
4408 let ctx = WaveCtx {
4409 carry_seats: false,
4410 pauses: Some((&self.pause, &self.interrupt)),
4411 run: &run_id,
4412 node: "vote",
4413 prompts: &prompts,
4414 cache: cache.as_deref(),
4415 round: None,
4416 };
4417 let results = ask_json_wave::<FinalVote>(
4418 jobs,
4419 Arc::clone(&self.sem),
4420 self.state.config.graph.retries,
4421 &self.roles.judge_roster,
4422 &ctx,
4423 &mut quota_losses,
4424 &mut self.state,
4425 &move |v: &FinalVote| match v.label() {
4426 Some(c) if allowed.contains(&c) => Ok(()),
4427 other => bail!("vote {other:?} is not one of {allowed:?}"),
4428 },
4429 )
4430 .await;
4431 if self.state.resource_block.is_some() {
4432 return self.stop_for_resource();
4433 }
4434 self.state.quota.extend(quota_losses);
4435
4436 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4437 let agent_id = seat.agent.clone();
4438 self.state.seats.insert(seat.key.clone(), seat);
4439 let initial = self
4440 .state
4441 .judgements
4442 .get(j)
4443 .and_then(|r| r.ranking.first().copied());
4444 let mut record = VoteRecord {
4445 judge: j + 1,
4446 agent: agent_id,
4447 vote: None,
4448 reason: String::new(),
4449 changed: false,
4450 };
4451 match res {
4452 Ok((v, _)) => {
4453 record.vote = v.label();
4454 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4455 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4456 self.state.event(
4457 "vote",
4458 format!(
4459 "judge {} voted {}{}",
4460 j + 1,
4461 record.vote.unwrap_or('?'),
4462 if record.changed { " (changed)" } else { "" }
4463 ),
4464 );
4465 }
4466 Err(e) => {
4467 self.state
4468 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4469 }
4470 }
4471 self.state.votes.push(record);
4472 self.state.save()?;
4473 }
4474 Ok(())
4475 }
4476
4477 fn tally(&mut self) -> Result<()> {
4480 if self.state.tally.is_some() {
4481 return Ok(());
4482 }
4483 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4484 let tops: Vec<char> = self
4485 .state
4486 .judgements
4487 .iter()
4488 .filter_map(|j| j.ranking.first().copied())
4489 .collect();
4490 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4491
4492 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4495 let mut cast: Vec<char> = Vec::new();
4496 for (i, j) in self.state.judgements.iter().enumerate() {
4497 let vote = self
4498 .state
4499 .votes
4500 .iter()
4501 .find(|v| v.judge == i + 1)
4502 .and_then(|v| v.vote)
4503 .or_else(|| j.ranking.first().copied());
4504 if let Some(v) = vote {
4505 *first_choice.entry(v).or_insert(0) += 1;
4506 cast.push(v);
4507 }
4508 }
4509
4510 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4511 for j in &self.state.judgements {
4512 let n = j.ranking.len();
4513 for (pos, label) in j.ranking.iter().enumerate() {
4514 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4515 }
4516 }
4517
4518 let best = first_choice.values().copied().max().unwrap_or(0);
4519 let mut leaders: Vec<char> = first_choice
4520 .iter()
4521 .filter(|(_, v)| **v == best)
4522 .map(|(k, _)| *k)
4523 .collect();
4524 let mut tie_break = None;
4525 if leaders.len() > 1 {
4526 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4527 let borda_leaders: Vec<char> = leaders
4528 .iter()
4529 .copied()
4530 .filter(|l| borda[l] == top_borda)
4531 .collect();
4532 tie_break = Some(if borda_leaders.len() == 1 {
4533 format!(
4534 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4535 leaders.len()
4536 )
4537 } else {
4538 format!(
4539 "{} way tie on both first-choice votes and Borda points, broken by label order",
4540 leaders.len()
4541 )
4542 });
4543 leaders = borda_leaders;
4544 leaders.sort_unstable();
4545 }
4546 let winner = *leaders
4547 .first()
4548 .or(viable.first())
4549 .context("no candidate to declare a winner from")?;
4550
4551 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4552 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4553 let deliberated = !self.state.deliberation.is_empty();
4554
4555 let quota_seats: std::collections::BTreeSet<&str> =
4559 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4560 let mut present = 0usize;
4561 for (i, j) in self.state.judgements.iter().enumerate() {
4562 if quota_seats.contains(j.seat.as_str()) {
4563 continue;
4564 }
4565 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4566 let voted = self
4567 .state
4568 .votes
4569 .iter()
4570 .any(|v| v.judge == i + 1 && v.vote.is_some());
4571 if ranked || voted {
4572 present += 1;
4573 }
4574 }
4575 let needs_quorum = viable.len() > 1;
4581 let judges_total = if needs_quorum {
4582 self.roles.judges.len()
4583 } else {
4584 0
4585 };
4586 let quorum = if needs_quorum {
4587 judges_total / 2 + 1
4588 } else {
4589 0
4590 };
4591 let met_quorum = !needs_quorum || present >= quorum;
4592 let uncontested = (!needs_quorum).then(|| {
4593 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4594 });
4595
4596 self.state.event(
4597 "tally",
4598 match &uncontested {
4599 Some(reason) => format!("winner {winner} — {reason}"),
4600 None => format!(
4601 "winner {winner} — votes {} | initial {} | {} changed | \
4602 {present}/{judges_total} judges{}",
4603 first_choice
4604 .iter()
4605 .map(|(k, v)| format!("{k}:{v}"))
4606 .collect::<Vec<_>>()
4607 .join(" "),
4608 if unanimous_initial {
4609 "unanimous"
4610 } else {
4611 "split"
4612 },
4613 changed_votes,
4614 if met_quorum {
4615 String::new()
4616 } else {
4617 format!(" — below quorum ({quorum} required)")
4618 },
4619 ),
4620 },
4621 );
4622 if !met_quorum {
4623 self.state.event(
4624 "stall",
4625 format!(
4626 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4627 the run stops here, resumable"
4628 ),
4629 );
4630 }
4631 self.state.tally = Some(Tally {
4632 first_choice,
4633 borda,
4634 winner,
4635 rankings: tops.len(),
4636 unanimous_initial,
4637 deliberated,
4638 changed_votes,
4639 unanimous_final,
4640 tie_break,
4641 judges: judges_total,
4642 present,
4643 quorum,
4644 met_quorum,
4645 uncontested,
4646 });
4647 self.state.status = if met_quorum {
4648 RunStatus::Reviewing
4649 } else {
4650 RunStatus::Stalled
4651 };
4652 self.state.save()?;
4653 Ok(())
4654 }
4655
4656 #[allow(clippy::too_many_lines)]
4677 async fn recover_stall(&mut self) -> Result<bool> {
4678 let run_id = self.state.id.clone();
4683 let prompts = self.state.config.prompts.clone();
4684 let quota_seats: BTreeSet<&str> =
4689 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4690 let absent: Vec<String> = self
4691 .state
4692 .judgements
4693 .iter()
4694 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4695 .map(|j| j.seat.clone())
4696 .collect();
4697 if absent.is_empty() {
4698 return Ok(false);
4699 }
4700 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4701 if viable.len() <= 1 {
4702 return Ok(false);
4703 }
4704 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4705 let language = self.state.config.graph.language.clone();
4706 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4707 let sessions = self.state.config.graph.sessions;
4708 let artifacts = agent::artifacts_dir(&self.state.dir());
4709 let root = self.state.worktree_root();
4710 let base_short = short(&self.state.base_commit);
4711 let candidates: Vec<Candidate> = viable.clone();
4712
4713 let mut positions: Vec<usize> = absent
4715 .iter()
4716 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4717 .collect();
4718 if positions.is_empty() {
4719 return Ok(false);
4720 }
4721 positions.sort_unstable();
4722 positions.dedup();
4723
4724 let mut judge_jobs = Vec::new();
4726 for &j in &positions {
4727 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4728 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4729 let seat_key = format!("judge-{}", j + 1);
4730 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4731 let seat = self.seat(&seat_key, &spec.id);
4732 judge_jobs.push(SeatJob {
4733 spec,
4734 seat,
4735 prompt: prompt::judge(
4736 &self.state.instruction,
4737 &views,
4738 self.roles.judges.len(),
4739 &base_short,
4740 &language,
4741 ),
4742 cwd: root.join(seat_key),
4743 timeout,
4744 allow_write: false,
4745 sessions,
4746 artifacts: artifacts.clone(),
4747 stem: format!("judge-{}-recover", j + 1),
4748 handover: None,
4749 });
4750 }
4751
4752 let labels_for_check = labels.clone();
4753 let mut judge_losses = Vec::new();
4754 let retries = self.state.config.graph.retries;
4755 let cache = self.state.config.cache_dir();
4756 let ctx = WaveCtx {
4757 carry_seats: false,
4758 pauses: Some((&self.pause, &self.interrupt)),
4759 run: &run_id,
4760 node: "judge",
4761 prompts: &prompts,
4762 cache: cache.as_deref(),
4763 round: None,
4764 };
4765 let results = ask_json_wave::<Ranking>(
4766 judge_jobs,
4767 Arc::clone(&self.sem),
4768 retries,
4769 &self.roles.judge_roster,
4770 &ctx,
4771 &mut judge_losses,
4772 &mut self.state,
4773 &move |r: &Ranking| r.validate(&labels_for_check),
4774 )
4775 .await;
4776 if self.state.resource_block.is_some() {
4777 self.stop_for_resource()?;
4778 return Ok(false);
4779 }
4780
4781 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4783 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4784 let agent_id = seat.agent.clone();
4785 self.state.seats.insert(seat.key.clone(), seat);
4786 let record = &mut self.state.judgements[j];
4787 match res {
4788 Ok((ranking, out)) => {
4789 record.agent = agent_id;
4790 record.ranking = ranking.normalized();
4791 record.reasons = ranking.reasons;
4792 record.strengths = ranking.strengths;
4793 record.confidence = ranking.confidence;
4794 record.failed = None;
4795 record.duration_ms = out.duration_ms;
4796 recovered.insert(j);
4797 self.state.event(
4798 "recover",
4799 format!("judge {} ranked again after the limit", j + 1),
4800 );
4801 }
4802 Err(e) => {
4803 self.state
4804 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4805 }
4806 }
4807 }
4808
4809 let mut vote_jobs = Vec::new();
4811 let mut vote_pos: Vec<usize> = Vec::new();
4812 for &j in &recovered {
4813 let seat_key = format!("judge-{}", j + 1);
4814 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4815 let seat = self.seat(&seat_key, &spec.id);
4816 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4817 let text = if has_context(&spec, &seat, sessions) {
4818 prompt::final_vote(&labels, &language)
4819 } else {
4820 full.clone()
4821 };
4822 vote_jobs.push(SeatJob {
4823 spec,
4824 seat,
4825 prompt: text,
4826 cwd: root.join(seat_key),
4827 timeout,
4828 allow_write: false,
4829 sessions,
4830 artifacts: artifacts.clone(),
4831 stem: format!("vote-judge-{}-recover", j + 1),
4832 handover: Some(full),
4833 });
4834 vote_pos.push(j);
4835 }
4836 let allowed = labels.clone();
4837 let mut vote_losses = Vec::new();
4838 let vote_retries = self.state.config.graph.retries;
4839 let vote_cache = self.state.config.cache_dir();
4840 let ctx = WaveCtx {
4841 carry_seats: false,
4842 pauses: Some((&self.pause, &self.interrupt)),
4843 run: &run_id,
4844 node: "vote",
4845 prompts: &prompts,
4846 cache: vote_cache.as_deref(),
4847 round: None,
4848 };
4849 let votes = ask_json_wave::<FinalVote>(
4850 vote_jobs,
4851 Arc::clone(&self.sem),
4852 vote_retries,
4853 &self.roles.judge_roster,
4854 &ctx,
4855 &mut vote_losses,
4856 &mut self.state,
4857 &move |v: &FinalVote| match v.label() {
4858 Some(c) if allowed.contains(&c) => Ok(()),
4859 other => bail!("vote {other:?} is not one of {allowed:?}"),
4860 },
4861 )
4862 .await;
4863 if self.state.resource_block.is_some() {
4864 self.stop_for_resource()?;
4865 return Ok(false);
4866 }
4867 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4868 let agent_id = seat.agent.clone();
4869 self.state.seats.insert(seat.key.clone(), seat);
4870 match res {
4871 Ok((v, _)) => {
4872 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4873 rec.vote = v.label();
4874 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4875 } else {
4876 self.state.votes.push(VoteRecord {
4877 judge: j + 1,
4878 agent: agent_id,
4879 vote: v.label(),
4880 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4881 changed: false,
4882 });
4883 }
4884 self.state.event(
4885 "recover",
4886 format!("judge {} voted again after the limit", j + 1),
4887 );
4888 }
4889 Err(e) => {
4890 self.state
4891 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4892 }
4893 }
4894 }
4895
4896 let recovered_keys: BTreeSet<String> = recovered
4900 .iter()
4901 .map(|&j| format!("judge-{}", j + 1))
4902 .collect();
4903 self.state
4904 .quota
4905 .retain(|q| !recovered_keys.contains(&q.seat));
4906 for loss in judge_losses.into_iter().chain(vote_losses) {
4910 if recovered_keys.contains(&loss.seat) {
4911 continue;
4912 }
4913 self.state.quota.retain(|q| q.seat != loss.seat);
4914 self.state.quota.push(loss);
4915 }
4916
4917 self.state.tally = None;
4919 self.tally()?;
4920 Ok(self
4921 .state
4922 .tally
4923 .as_ref()
4924 .map(|t| t.met_quorum)
4925 .unwrap_or(false))
4926 }
4927
4928 async fn graft(&mut self) -> Result<()> {
4944 let Err(e) = self.graft_inner().await else {
4945 return Ok(());
4946 };
4947 if e.is::<crate::repo_lock::ResourceBlocked>() {
4948 return Err(e);
4949 }
4950 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4951 self.state.status = RunStatus::Blocked;
4952 self.state.event("graft", format!("blocked: {e:#}"));
4953 self.state.save()?;
4954 return Err(e);
4955 }
4956 if let Some(rec) = self.state.graft.clone()
4961 && !rec.outcome.settled()
4962 {
4963 if let Some(winner) = self.state.winner().cloned() {
4964 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4965 if re.is::<crate::repo_lock::ResourceBlocked>() {
4966 return Err(re);
4967 }
4968 self.state.status = RunStatus::Blocked;
4969 self.state.event("graft", format!("blocked: {re:#}"));
4970 self.state.save()?;
4971 return Err(re);
4972 }
4973 }
4974 if let Some(g) = self.state.graft.as_mut() {
4975 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4976 }
4977 }
4978 self.state.event("graft", format!("skipped: {e:#}"));
4979 if let Err(e) = self.state.save() {
4980 tracing::warn!("save after graft: {e:#}");
4981 }
4982 Ok(())
4983 }
4984
4985 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4988 let _coordination = self.coordinate(wt).await?;
4989 crate::recovery::restore_repair(wt, pre)
4990 .await
4991 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4992 }
4993
4994 fn graft_skipped(&mut self, why: &str) {
4996 let message = format!("skipped: {why}");
4997 if !self
4998 .state
4999 .events
5000 .iter()
5001 .any(|e| e.node == "graft" && e.message == message)
5002 {
5003 self.state.event("graft", message);
5004 }
5005 }
5006
5007 async fn graft_inner(&mut self) -> Result<()> {
5008 if let Some(rec) = self.state.graft.clone()
5011 && !rec.outcome.settled()
5012 {
5013 if self.state.reviews.is_empty()
5017 && let Some(winner) = self.state.winner().cloned()
5018 {
5019 self.graft_restore(&winner.worktree, &rec.pre).await?;
5020 }
5021 if rec.outcome == GraftOutcome::WaitingForRestore {
5022 self.state.graft = None;
5023 self.state.save()?;
5024 } else {
5025 if let Some(g) = self.state.graft.as_mut() {
5026 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5027 }
5028 self.state.event(
5029 "graft",
5030 "rolled back: the previous round was interrupted; the winner is untouched",
5031 );
5032 self.state.save()?;
5033 return Ok(());
5034 }
5035 }
5036 let plan = match graft::plan(&self.state) {
5037 Ok(plan) => plan,
5038 Err(skip) => {
5039 if skip != graft::Skip::AlreadyDone {
5040 self.graft_skipped(skip.reason());
5041 self.state.save()?;
5042 }
5043 return Ok(());
5044 }
5045 };
5046 let Some(winner) = self.state.winner().cloned() else {
5047 return Ok(());
5048 };
5049 let wt = winner.worktree.clone();
5050 if !git::is_clean(&wt).await? {
5051 self.graft_skipped("the winner's worktree has uncommitted changes");
5052 self.state.save()?;
5053 return Ok(());
5054 }
5055 let pre = git::rev_parse(&wt, "HEAD").await?;
5056 self.state.graft = Some(GraftRecord {
5059 pre: pre.clone(),
5060 post: None,
5061 lines: None,
5062 limit: None,
5063 agent: None,
5064 outcome: GraftOutcome::Running,
5065 requested: graft::requested(&plan),
5066 adoption: None,
5067 adoption_error: None,
5068 });
5069 self.state.event(
5070 "graft",
5071 format!(
5072 "porting strengths of candidate(s) {} onto {}",
5073 plan.strengths
5074 .iter()
5075 .map(|(l, _, _)| l.to_string())
5076 .collect::<Vec<_>>()
5077 .join(", "),
5078 plan.winner
5079 ),
5080 );
5081 self.state.save()?;
5082
5083 let fix_prompt = prompt::graft(
5084 &self.state.instruction,
5085 plan.winner,
5086 &plan.strengths,
5087 &self.state.config.graph.language,
5088 );
5089 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5090 let sessions = self.state.config.graph.sessions;
5091 let artifacts = agent::artifacts_dir(&self.state.dir());
5092 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5093 let (_, seat, out) = self
5094 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5095 SeatJob {
5096 prompt: fix_prompt.clone(),
5097 spec,
5098 seat,
5099 cwd: wt.clone(),
5100 timeout,
5101 allow_write: true,
5102 sessions,
5103 artifacts: artifacts.clone(),
5104 stem: "graft".to_owned(),
5105 handover: None,
5106 }
5107 })
5108 .await;
5109 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5110 self.state.graft.as_mut().expect("graft record").outcome =
5113 GraftOutcome::WaitingForRestore;
5114 return self.stop_for_resource();
5115 }
5116 let mut why: Option<String> = None;
5117 match &out {
5118 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5119 why = Some("the fixer produced nothing usable".to_owned());
5120 }
5121 AgentOutcome::Ok(_) => {}
5122 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5123 AgentOutcome::Quota(o) => {
5124 self.state.quota.push(QuotaLoss::from_report(
5125 seat.key.clone(),
5126 "graft",
5127 o.quota.as_ref(),
5128 ));
5129 why = Some("rate limited (quota)".to_owned());
5130 }
5131 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5132 why = Some(format!(
5133 "the fixer failed: {}",
5134 e.lines()
5135 .next()
5136 .unwrap_or("")
5137 .chars()
5138 .take(160)
5139 .collect::<String>()
5140 ));
5141 }
5142 }
5143 let agent_id = seat.agent.clone();
5144 self.state.seats.insert(seat.key.clone(), seat);
5145
5146 let mut post = None;
5147 let mut lines = None;
5148 let _coordination = self.coordinate(&wt).await?;
5149 if why.is_none() {
5150 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5151 if let Ok(r) = &rescued {
5152 self.state.note_withheld("graft", &r.withheld);
5153 }
5154 let head = git::rev_parse(&wt, "HEAD").await?;
5155 let dirty = match &rescued {
5159 Ok(r) => git::status_porcelain(&wt)
5160 .await
5161 .map(|st| {
5162 st.lines().any(|l| {
5163 !r.withheld
5164 .iter()
5165 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5166 })
5167 })
5168 .unwrap_or(true),
5169 Err(_) => true,
5170 };
5171 if dirty {
5172 why = Some("the fixer left changes that could not be committed".to_owned());
5173 } else if head == pre {
5174 why = Some("the fixer changed nothing".to_owned());
5175 } else {
5176 lines = git::changed_lines(&wt, &pre, &head).await;
5177 post = Some(head);
5178 }
5179 }
5180 let (adoption, adoption_error) = match &out {
5181 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5182 _ => (None, None),
5183 };
5184 let requested = graft::requested(&plan);
5185 let outcome = match why {
5186 None => {
5187 self.state.event(
5188 "graft",
5189 format!(
5190 "applied: {}",
5191 lines.map_or_else(
5192 || "changed lines unavailable".to_owned(),
5193 |n| format!("{n} changed lines")
5194 )
5195 ),
5196 );
5197 GraftOutcome::Applied
5198 }
5199 Some(why) => {
5200 crate::recovery::restore_repair(&wt, &pre)
5203 .await
5204 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5205 self.state.event(
5206 "graft",
5207 format!("rolled back: {why}; the winner is untouched"),
5208 );
5209 GraftOutcome::RolledBack(why)
5210 }
5211 };
5212 self.state.graft = Some(GraftRecord {
5213 pre,
5214 post,
5215 lines,
5216 limit: None,
5217 agent: Some(agent_id),
5218 outcome,
5219 requested,
5220 adoption,
5221 adoption_error,
5222 });
5223 self.state.save()?;
5224 Ok(())
5225 }
5226
5227 async fn fold_losers(&mut self) -> Result<()> {
5230 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5231 return Ok(());
5232 };
5233 if !self.state.candidate_preservation.is_empty() {
5234 bail!("candidate preservation incomplete; refusing cleanup");
5235 }
5236 let repo = self.state.repo.clone();
5237 let _coordination = self.coordinate(&repo).await?;
5238 let mut folded = Vec::new();
5239 for i in 0..self.state.candidates.len() {
5240 let c = &self.state.candidates[i];
5241 if c.label == winner || c.folded {
5242 continue;
5243 }
5244 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5245 remove_preserved_worktree(&repo, &wt).await?;
5246 if git::branch_exists(&repo, &branch).await?
5247 && !git::branch_delete(&repo, &branch).await?
5248 {
5249 bail!("candidate branch removal refused");
5250 }
5251 self.state.candidates[i].folded = true;
5252 self.state.save()?;
5253 folded.push(label.to_string());
5254 }
5255 let root = self.state.worktree_root();
5257 for j in 1..=self.roles.judges.len() {
5258 let wt = root.join(format!("judge-{j}"));
5259 if wt.exists() {
5260 remove_preserved_worktree(&repo, &wt).await?;
5261 }
5262 }
5263 if self.state.config.graph.advise {
5266 for k in 1..=self.state.config.graph.advisors {
5267 let wt = root.join(format!("advisor-{k}"));
5268 if wt.exists() {
5269 remove_preserved_worktree(&repo, &wt).await?;
5270 }
5271 }
5272 }
5273 if !folded.is_empty() {
5274 self.state
5275 .event("fold", format!("folded candidates {}", folded.join(", ")));
5276 self.state.save()?;
5277 }
5278 Ok(())
5279 }
5280
5281 async fn hold_base_sync(&mut self) -> Result<bool> {
5284 self.hold_base_sync_as(None).await
5285 }
5286
5287 async fn hold_base_sync_kind(
5288 &mut self,
5289 kind: crate::base_sync_recovery::StopKind,
5290 ) -> Result<()> {
5291 self.hold_base_sync_as(Some(kind)).await?;
5292 Ok(())
5293 }
5294
5295 async fn hold_base_sync_as(
5296 &mut self,
5297 kind: Option<crate::base_sync_recovery::StopKind>,
5298 ) -> Result<bool> {
5299 if !self.state.base_sync_blocked() {
5300 self.state.base_sync_recovery.stop = None;
5301 return Ok(false);
5302 }
5303 let head = match self.state.winner() {
5304 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5305 .await
5306 .unwrap_or_default(),
5307 None => String::new(),
5308 };
5309 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5310 && !authorization.consumed
5311 && authorization.stop.head != head
5312 {
5313 authorization.refuse(format!(
5314 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5315 authorization.stop.head
5316 ));
5317 }
5318 self.state.record_base_sync_stop(&head);
5319 if let Some(kind) = kind {
5320 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5321 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5322 stop.kind = kind;
5323 }
5324 }
5325 self.state.save()?;
5326 Ok(true)
5327 }
5328
5329 async fn sync_to_base(&mut self) -> Result<()> {
5371 if self.state.status == RunStatus::AlreadyInBase {
5372 return Ok(());
5373 }
5374 self.hold_base_sync().await?;
5375 let conflicted = self
5376 .state
5377 .base_sync
5378 .as_ref()
5379 .is_some_and(|s| s.conflict.is_some());
5380 let Some(winner) = self.state.winner().cloned() else {
5381 return Ok(());
5382 };
5383
5384 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5385
5386 let repo = self.state.repo.clone();
5387 let remote = self.state.config.merge.remote.clone();
5388 let base_branch = self.state.base_branch.clone();
5389 let tracking = format!("{remote}/{base_branch}");
5390
5391 let fetched_base =
5392 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5393 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5398 return Ok(());
5399 };
5400
5401 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5402 self.state.ensure_base_episode();
5403 let mut pending = self
5404 .state
5405 .base_sync_recovery
5406 .operations
5407 .last()
5408 .filter(|o| !o.finished)
5409 .cloned();
5410 let scratch = self.state.dir().join("base-sync");
5411 if git::rebase_in_progress(&scratch).await {
5412 if pending.is_none()
5415 && self.state.base_sync_recovery.operations.is_empty()
5416 && !conflicted
5417 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5418 .await
5419 .ok()
5420 .as_deref()
5421 == Some(head.as_str())
5422 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5423 && git::is_clean(&winner.worktree).await?
5424 {
5425 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5426 if self.state.base_sync.is_none() {
5427 self.state.base_sync = Some(BaseSync {
5428 tip: saved_tip.clone(),
5429 behind: 0,
5430 attempts: 0,
5431 conflict: None,
5432 already_in: None,
5433 });
5434 }
5435 self.state.reserve_base_operation(&head, &saved_tip);
5436 self.state.event(
5437 "base-recovery",
5438 "imported standing legacy rebase as a spent synchronization reservation",
5439 );
5440 self.state.save()?;
5441 pending = self.state.base_sync_recovery.operations.last().cloned();
5442 }
5443 let expected = pending.as_ref().map(|op| op.tip.as_str());
5444 if expected.is_none()
5445 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5446 {
5447 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5448 if self.state.base_sync.is_none() {
5449 self.state.base_sync = Some(BaseSync {
5450 tip: tip.clone(),
5451 behind: 0,
5452 attempts: 0,
5453 conflict: None,
5454 already_in: None,
5455 });
5456 }
5457 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5458 self.state.event("land", why);
5459 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5460 .await?;
5461 return Ok(());
5462 }
5463 }
5464 if let Some(op) = &pending {
5465 let mut replay_proven =
5467 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5468 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5469 let wait = repository_wait(&self.state, None);
5470 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5471 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5472 Ok(true) => {
5473 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5474 self.state.save()?;
5475 git::worktree_remove(&repo, &scratch).await.ok();
5476 replay_proven = true;
5477 }
5478 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5479 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5480 }
5481 }
5482 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5483 if self.state.foreign_checkout.is_some() {
5484 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5485 .await
5486 .ok()
5487 .as_deref()
5488 == Some(op.head.as_str())
5489 && git::rev_parse(&repo, &format!("{head}^2"))
5490 .await
5491 .ok()
5492 .as_deref()
5493 == Some(op.tip.as_str());
5494 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5495 .await
5496 .is_ok_and(|m| m.is_empty())
5497 && let Ok(added) =
5498 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5499 {
5500 replay_proven = true;
5501 for commit in added {
5502 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5503 replay_proven = false;
5504 break;
5505 }
5506 }
5507 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5508 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5509 }
5510 }
5511 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5515 || (head != op.head
5516 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5517 .await?
5518 .is_empty())
5519 || (!git::is_clean(&winner.worktree).await?
5520 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5521 .await
5522 .is_ok_and(|o| o.ok()))
5523 || !replay_proven
5524 {
5525 let why = format!(
5526 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5527 op.id
5528 );
5529 let sync = self
5530 .state
5531 .base_sync
5532 .as_mut()
5533 .context("reserved synchronization has no lifetime accounting")?;
5534 sync.conflict = Some(why.clone());
5535 self.state.event("land", why);
5536 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5537 .await?;
5538 return Ok(());
5539 }
5540 if head != op.head {
5541 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5542 }
5543 tip = op.tip.clone();
5544 }
5545 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5546 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5547 if pending.is_none()
5548 && !conflicted
5549 && fetched_base
5550 && self
5551 .state
5552 .base_sync_recovery
5553 .episodes
5554 .last()
5555 .is_some_and(|e| e.boundary.is_some())
5556 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5557 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5558 && owner.runs.last() == Some(&self.state.id)
5559 && owner.status != crate::queue::TaskStatus::Done
5560 && !owner.fresh_start
5561 && owner.review_branch.is_none()
5562 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5563 && let Some(approval) = owner.resumption.as_ref()
5564 && approval.instruction == owner.instruction
5565 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5566 && git::is_clean(&winner.worktree).await?
5567 && self.state.renew_base_episode(&task, approval, &head, &tip)
5568 {
5569 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5570 self.state.save()?;
5571 }
5572
5573 if (behind > 0 || conflicted || head == tip)
5582 && self
5583 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5584 .await?
5585 {
5586 return Ok(());
5587 }
5588 if conflicted && self.state.base_recovery_pending() {
5589 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5590 let owner =
5591 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5592 let owner_valid = owner.as_ref().is_ok_and(|t| {
5593 t.runs.last() == Some(&self.state.id)
5594 && t.status != crate::queue::TaskStatus::Done
5595 && !t.fresh_start
5596 && t.review_branch.is_none()
5597 });
5598 let invalidation = if !owner_valid {
5599 Some("latest task ownership changed")
5600 } else if !owner.as_ref().is_ok_and(|t| {
5601 authorization
5602 .generation
5603 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5604 }) {
5605 Some("owner approval generation changed")
5606 } else if let Some(scope) = &authorization.scope {
5607 if scope.tip != tip {
5608 Some("fetched base changed")
5609 } else if !self
5610 .state
5611 .base_sync_recovery
5612 .episodes
5613 .last()
5614 .is_some_and(|e| e.id == scope.episode)
5615 {
5616 Some("active synchronization interval changed")
5617 } else if !owner
5618 .as_ref()
5619 .is_ok_and(|t| t.instruction == scope.instruction)
5620 {
5621 Some("task instruction changed")
5622 } else {
5623 None
5624 }
5625 } else {
5626 None
5627 };
5628 if let Some(reason) = invalidation {
5629 let authorization = self
5630 .state
5631 .base_sync_recovery
5632 .authorizations
5633 .last_mut()
5634 .unwrap();
5635 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5636 self.state.save()?;
5637 return Ok(());
5638 }
5639 if !fetched_base
5640 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5641 || !git::is_clean(&winner.worktree).await?
5642 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5643 {
5644 return Ok(());
5645 }
5646 }
5647 let recovery = conflicted && self.state.consume_base_recovery(&head);
5648 if conflicted {
5649 self.state.save()?;
5651 if !recovery {
5652 return Ok(());
5653 }
5654 }
5655
5656 if behind == 0 {
5657 if let Some(from) = self
5666 .state
5667 .rebase_fixes
5668 .iter()
5669 .rev()
5670 .find_map(|r| r.from.clone())
5671 && from != head
5672 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5673 .await
5674 .is_ok_and(|o| o.ok())
5675 {
5676 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5677 }
5678 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5684 if let Some(why) = &conflict {
5685 self.state.status = RunStatus::Blocked;
5686 self.state.event("land", why.clone());
5687 }
5688 self.state.base_sync = Some(BaseSync {
5689 tip,
5690 behind: 0,
5691 attempts,
5692 conflict,
5693 already_in: None,
5694 });
5695 if pending.is_some() {
5696 self.state.finish_base_operation(Some(&head));
5697 }
5698 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5699 .await?;
5700 self.state.save()?;
5701 return Ok(());
5702 }
5703
5704 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5705 let why = format!(
5706 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5707 synchronization(s); synchronizing again would only race it",
5708 winner.branch
5709 );
5710 self.state.status = RunStatus::Blocked;
5711 self.state.base_sync = Some(BaseSync {
5712 tip,
5713 behind,
5714 attempts,
5715 conflict: Some(why.clone()),
5716 already_in: None,
5717 });
5718 self.state.event("land", why);
5719 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5720 .await?;
5721 self.state.save()?;
5722 return Ok(());
5723 }
5724
5725 if self.state.foreign_checkout.is_some() {
5726 let wait = repository_wait(&self.state, None);
5729 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5730 if self.state.base_sync.is_none() {
5731 self.state.base_sync = Some(BaseSync {
5732 tip: tip.clone(),
5733 behind,
5734 attempts: 0,
5735 conflict: None,
5736 already_in: None,
5737 });
5738 }
5739 self.state.reserve_base_operation(&head, &tip);
5740 self.state.save()?;
5741 let merged = git::git_raw(
5742 &winner.worktree,
5743 &[
5744 "-c",
5745 "core.hooksPath=/dev/null",
5746 "merge",
5747 "--no-ff",
5748 "--no-autostash",
5749 "--no-edit",
5750 "-m",
5751 &format!("Merge {tracking} before verifying"),
5752 &tip,
5753 ],
5754 )
5755 .await?;
5756 let conflict = if merged.ok() {
5757 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5758 self.state.event(
5759 "land",
5760 format!("merged {tracking} into {} before verifying", winner.branch),
5761 );
5762 None
5763 } else {
5764 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5765 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5766 }
5767 let why = format!(
5768 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5769 winner.branch,
5770 merged.stderr.chars().take(600).collect::<String>()
5771 );
5772 self.state.status = RunStatus::Blocked;
5773 self.state.event("land", &why);
5774 Some(why)
5775 };
5776 let synced = conflict.is_none();
5777 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5778 self.state
5779 .finish_base_operation(synced.then_some(fresh.as_str()));
5780 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5781 self.state.base_sync = Some(BaseSync {
5782 tip,
5783 behind: if synced { 0 } else { behind },
5784 attempts,
5785 conflict,
5786 already_in: None,
5787 });
5788 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5789 .await?;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793
5794 self.state.event(
5795 "land",
5796 format!(
5797 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5798 winner.branch
5799 ),
5800 );
5801 self.state.save()?;
5802
5803 let branch_tracking = format!("{remote}/{}", winner.branch);
5809 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5810 let remote_tip = if self.state.task_pr_branch().is_none()
5811 && matches!(&fetched_branch, Ok(o) if o.ok())
5812 {
5813 git::rev_parse(&repo, &branch_tracking).await.ok()
5814 } else {
5815 None
5816 };
5817 if let Some(theirs) = &remote_tip
5821 && !git::is_ancestor(&repo, theirs, &head).await
5822 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5823 .await
5824 .is_ok_and(|missing| missing.is_empty())
5825 {
5826 let why = format!(
5827 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5828 them",
5829 short(theirs),
5830 winner.branch
5831 );
5832 self.state.status = RunStatus::Blocked;
5833 self.state.base_sync = Some(BaseSync {
5834 tip,
5835 behind,
5836 attempts,
5837 conflict: Some(why.clone()),
5838 already_in: None,
5839 });
5840 self.state.event("land", why);
5841 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5842 .await?;
5843 self.state.save()?;
5844 return Ok(());
5845 }
5846
5847 if self.state.base_sync.is_none() {
5848 self.state.base_sync = Some(BaseSync {
5849 tip: tip.clone(),
5850 behind,
5851 attempts: 0,
5852 conflict: None,
5853 already_in: None,
5854 });
5855 }
5856 self.state.reserve_base_operation(&head, &tip);
5857 self.state.save()?;
5858 let scratch = self.state.dir().join("base-sync");
5859 let rebased =
5860 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5861 .await
5862 {
5863 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5864 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5865 Err(e) => Err(e),
5866 };
5867 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5868 let stop_kind = match &rebased {
5869 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5870 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5871 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5872 };
5873 match rebased {
5874 Ok(None) => {
5875 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5879 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5880 self.state.finish_base_operation(Some(&fresh));
5881 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5882 let mut conflict = None;
5883 if let Some(pinned) = &remote_tip {
5884 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5885 match pushed {
5886 Ok(o) if o.ok() => self.state.event(
5887 "land",
5888 format!("pushed rebased {} to {remote}", winner.branch),
5889 ),
5890 Ok(o) => {
5891 conflict = Some(format!(
5892 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5893 winner.branch,
5894 short(pinned),
5895 o.stderr.chars().take(600).collect::<String>()
5896 ));
5897 }
5898 Err(e) => {
5899 conflict = Some(format!(
5900 "rebased {} locally but could not push it: {e:#}",
5901 winner.branch
5902 ));
5903 }
5904 }
5905 }
5906 if let Some(why) = &conflict {
5907 self.state.status = RunStatus::Blocked;
5908 self.state.event("land", why.clone());
5909 }
5910 self.state.base_sync = Some(BaseSync {
5911 tip: tip.clone(),
5912 behind: 0,
5913 attempts,
5914 conflict,
5915 already_in: None,
5916 });
5917 self.state
5918 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5919 }
5920 Ok(Some(conflict)) => {
5921 self.state.finish_base_operation(None);
5922 let why = format!(
5923 "{} conflicts with {tracking} and did not rebase: {}",
5924 winner.branch,
5925 conflict.chars().take(600).collect::<String>()
5926 );
5927 self.state.status = RunStatus::Blocked;
5928 self.state.base_sync = Some(BaseSync {
5929 tip,
5930 behind,
5931 attempts,
5932 conflict: Some(why.clone()),
5933 already_in: None,
5934 });
5935 self.state.event("land", why);
5936 }
5937 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5941 self.state.save()?;
5942 return Ok(());
5943 }
5944 Err(e) => {
5945 self.state.finish_base_operation(None);
5946 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5947 self.state.status = RunStatus::Blocked;
5948 self.state.base_sync = Some(BaseSync {
5949 tip,
5950 behind,
5951 attempts,
5952 conflict: Some(why.clone()),
5953 already_in: None,
5954 });
5955 self.state.event("land", why);
5956 }
5957 }
5958 if !self.state.base_sync_blocked() {
5959 self.state.base_sync_recovery.stop = None;
5960 }
5961 self.hold_base_sync_kind(stop_kind).await?;
5962 self.state.save()?;
5963 Ok(())
5964 }
5965
5966 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5972 if self.state.task_pr_branch().is_some() {
5975 return None;
5976 }
5977 let repo = self.state.repo.clone();
5978 let remote = self.state.config.merge.remote.clone();
5979 let fetched = git::fetch(&repo, &remote, branch).await;
5980 if !matches!(&fetched, Ok(o) if o.ok()) {
5981 return None;
5982 }
5983 let branch_tracking = format!("{remote}/{branch}");
5984 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5985 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5986 return None;
5987 }
5988 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5989 .await
5990 .is_ok_and(|missing| missing.is_empty())
5991 {
5992 return Some(format!(
5993 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5994 them",
5995 short(&theirs)
5996 ));
5997 }
5998 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5999 Ok(o) if o.ok() => {
6000 self.state
6001 .event("land", format!("pushed rebased {branch} to {remote}"));
6002 None
6003 }
6004 Ok(o) => Some(format!(
6005 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6006 someone may have pushed): {}",
6007 short(&theirs),
6008 o.stderr.chars().take(600).collect::<String>()
6009 )),
6010 Err(e) => Some(format!(
6011 "{branch} is rebased locally but could not be pushed: {e:#}"
6012 )),
6013 }
6014 }
6015
6016 async fn settle_already_in(
6033 &mut self,
6034 branch: &str,
6035 tip: &str,
6036 head: &str,
6037 attempts: usize,
6038 behind: usize,
6039 ) -> Result<bool> {
6040 let repo = self.state.repo.clone();
6041 let remote = self.state.config.merge.remote.clone();
6042 let start = self.state.base_commit.clone();
6043 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6044 Ok(Some(e)) => e,
6045 Ok(None) => return Ok(false),
6046 Err(e) => {
6047 tracing::warn!("already-in-base check for {branch}: {e:#}");
6048 return Ok(false);
6049 }
6050 };
6051 let mut verified = vec![head.to_owned()];
6052 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6053 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6054 if matches!(&fetched, Ok(o) if o.ok())
6055 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6056 && theirs != head
6057 {
6058 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6059 Ok(Some(_)) => verified.push(theirs),
6060 _ => return Ok(false),
6061 }
6062 }
6063 let base_branch = self.state.base_branch.clone();
6064 let message = format!(
6065 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6066 land",
6067 evidence.names(),
6068 evidence.proof.as_str()
6069 );
6070 let closed =
6071 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6072 .await;
6073 match closed {
6074 Ok(Ok(url)) => self
6075 .state
6076 .event("land", format!("closed {url}: superseded on {base_branch}")),
6077 Ok(Err(why)) => self
6078 .state
6079 .event("land", format!("did not close a pull request: {why}")),
6080 Err(e) => {
6081 let why = format!(
6082 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6083 not be closed ({e:#}); resume to retry"
6084 );
6085 self.state.status = RunStatus::Blocked;
6086 self.state.base_sync = Some(BaseSync {
6087 tip: tip.to_owned(),
6088 behind,
6089 attempts,
6090 conflict: Some(why.clone()),
6091 already_in: None,
6092 });
6093 self.state.event("land", why);
6094 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6095 .await?;
6096 self.state.save()?;
6097 return Ok(true);
6098 }
6099 }
6100 self.state.status = RunStatus::AlreadyInBase;
6101 self.state.base_sync_recovery.stop = None;
6102 self.state.base_sync = Some(BaseSync {
6103 tip: tip.to_owned(),
6104 behind,
6105 attempts,
6106 conflict: None,
6107 already_in: Some(evidence),
6108 });
6109 self.state.event("land", message);
6110 self.state.save()?;
6111 self.settle_questions();
6112 Ok(true)
6113 }
6114
6115 fn landing_base(&self) -> String {
6125 self.state
6126 .base_sync
6127 .as_ref()
6128 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6129 }
6130
6131 pub async fn fix_selected(
6164 &mut self,
6165 ids: &[String],
6166 reason: &str,
6167 allow_stale: bool,
6168 ) -> Result<()> {
6169 let reason = reason.trim();
6170 if reason.is_empty() {
6171 bail!("a fix request needs a reason — that is the operator's own record of why");
6172 }
6173 if ids.is_empty() {
6174 bail!("no finding id given");
6175 }
6176 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6177 bail!(
6178 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6179 has already concluded — can be given a targeted fix. A run still \
6180 in progress should simply be resumed; a `merged` run's branch has \
6181 already landed, so its answer is a fresh `magi review <branch>`, \
6182 not reopening this run's own record",
6183 self.state.id,
6184 self.state.status.as_str()
6185 );
6186 }
6187 let Some(winner) = self.state.winner().cloned() else {
6188 bail!("run {} has no winning candidate to fix", self.state.id);
6189 };
6190 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6191 bail!(
6192 "branch `{}` no longer exists; this run cannot be extended",
6193 winner.branch
6194 );
6195 }
6196 let home = crate::run::home();
6197 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6198 bail!(
6199 "run {} is currently being worked on by another magi process",
6200 self.state.id
6201 );
6202 }
6203 let _claim = FixClaim::acquire(&self.state.dir())?;
6209
6210 let mut seen = BTreeSet::new();
6214 let mut findings = Vec::new();
6215 let mut missing = Vec::new();
6216 for id in ids {
6217 if !seen.insert(id.clone()) {
6218 continue;
6219 }
6220 match self.state.finding(id) {
6221 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6222 id: f.id.clone(),
6223 severity: f.severity,
6224 reviewer_vote: rec.vote,
6225 round: round.round,
6226 round_head: round.head.clone(),
6227 reviewer: rec.reviewer,
6228 agent: rec.agent.clone(),
6229 file: f.file.clone(),
6230 line: f.line,
6231 title: f.title.clone(),
6232 detail: f.detail.clone(),
6233 outcome: OperatorFixOutcome::Pending,
6234 }),
6235 None => missing.push(id.clone()),
6236 }
6237 }
6238 if !missing.is_empty() {
6239 bail!(
6240 "unknown finding id(s): {}; nothing was changed",
6241 missing.join(", ")
6242 );
6243 }
6244
6245 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6246 let stale_details: Vec<(String, String)> = findings
6247 .iter()
6248 .filter(|f| f.round_head != head_at_request)
6249 .map(|f| (f.id.clone(), f.round_head.clone()))
6250 .collect();
6251 let stale = !stale_details.is_empty();
6252 if stale && !allow_stale {
6253 bail!(
6254 "the branch has moved since some finding(s) were raised — {} — now \
6255 at {}; pass --allow-stale to fix anyway, or re-run review first",
6256 stale_details
6257 .iter()
6258 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6259 .collect::<Vec<_>>()
6260 .join(", "),
6261 short(&head_at_request)
6262 );
6263 }
6264
6265 let request = OperatorFixRequest {
6266 requested_at: Timestamp::now(),
6267 reason: reason.to_owned(),
6268 findings,
6269 head_at_request: head_at_request.clone(),
6270 allow_stale,
6271 stale,
6272 fix: None,
6273 result_head: None,
6274 follow_up_review_run: None,
6275 };
6276 self.state.event(
6277 "fix",
6278 format!(
6279 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6280 request.findings.len(),
6281 request
6282 .findings
6283 .iter()
6284 .map(|f| f.id.as_str())
6285 .collect::<Vec<_>>()
6286 .join(", "),
6287 ),
6288 );
6289 self.state.operator_fixes.push(request);
6296 self.state.save()?;
6297 let request_index = self.state.operator_fixes.len() - 1;
6298
6299 let repo = self.state.repo.clone();
6308 let coordination = self.coordinate(&repo).await?;
6309 if winner.worktree.exists() {
6310 let dirty = git::git(
6313 &winner.worktree,
6314 &["status", "--porcelain", "--untracked-files=all"],
6315 )
6316 .await?;
6317 let only_withheld = dirty.lines().all(|l| {
6318 l.strip_prefix("?? ")
6319 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6320 });
6321 if !only_withheld {
6322 bail!(
6323 "`{}` has uncommitted changes; refusing to touch it — commit or \
6324 discard them first",
6325 winner.worktree.display()
6326 );
6327 }
6328 git::worktree_remove(&self.state.repo, &winner.worktree)
6329 .await
6330 .ok();
6331 }
6332 let fix_worktree = self.state.worktree_root().join("operator-fix");
6333 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6334 git::git(
6335 &self.state.repo,
6336 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6337 )
6338 .await
6339 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6340 if !git::is_clean(&fix_worktree).await? {
6341 git::worktree_remove(&self.state.repo, &fix_worktree)
6342 .await
6343 .ok();
6344 bail!(
6345 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6346 winner.branch
6347 );
6348 }
6349 let repo_for_setup = self.state.repo.clone();
6350 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6351 Ok(()) => {
6352 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6353 }
6354 Err(e) => Err(e),
6355 };
6356 if let Err(e) = setup {
6357 git::worktree_remove(&self.state.repo, &fix_worktree)
6358 .await
6359 .ok();
6360 return Err(e);
6361 }
6362
6363 self.state.resource_block = None;
6366
6367 let run_id = self.state.id.clone();
6368 let prompts = self.state.config.prompts.clone();
6369 let language = self.state.config.graph.language.clone();
6370 let sessions = self.state.config.graph.sessions;
6371 let artifacts = agent::artifacts_dir(&self.state.dir());
6372 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6373 .findings
6374 .iter()
6375 .map(|f| Finding {
6376 id: f.id.clone(),
6377 severity: f.severity,
6378 file: f.file.clone(),
6379 line: f.line,
6380 title: f.title.clone(),
6381 detail: f.detail.clone(),
6382 })
6383 .collect();
6384 let fix_prompt = prompt::operator_fix_in(
6385 &self.state.instruction,
6386 &finding_list,
6387 reason,
6388 &stale_details,
6389 &head_at_request,
6390 &language,
6391 &self.state.config.graph.github_language,
6392 );
6393 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6394 drop(coordination);
6395 let (job, seat, out) = self
6396 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6397 prompt: fix_prompt.clone(),
6398 spec,
6399 seat,
6400 cwd: fix_worktree.clone(),
6401 timeout,
6402 allow_write: true,
6403 sessions,
6404 artifacts: artifacts.clone(),
6405 stem: "operator-fix".to_owned(),
6406 handover: None,
6407 })
6408 .await;
6409 let agent_id = seat.agent.clone();
6410
6411 let mut fix = FixRecord {
6412 agent: agent_id,
6413 addressed: Vec::new(),
6414 rejected: Vec::new(),
6415 notes: String::new(),
6416 committed: false,
6417 failed: None,
6418 duration_ms: 0,
6419 continuation: None,
6420 };
6421 let mut final_seat = seat.clone();
6422 match out {
6423 AgentOutcome::Ok(o) => {
6424 fix.duration_ms = o.duration_ms;
6425 let parsed = verdict::extract_json::<FixReport>(&o.text);
6426 let incomplete_reason = match &parsed {
6427 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6428 "the reply parsed, but it reported a command whose own CLI \
6429 never confirmed an exit status"
6430 .to_owned(),
6431 ),
6432 Ok(_) => None,
6433 Err(e) => Some(e.to_string()),
6434 };
6435 match incomplete_reason {
6436 None => {
6437 let report = parsed.expect("checked Ok above");
6438 fix.addressed = report.addressed;
6439 fix.rejected = report.rejected;
6440 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6441 }
6442 Some(reason) => {
6443 let (resumed_seat, resolved, failure, cont) = self
6444 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6445 .await;
6446 fix.duration_ms += cont.cumulative_wait_ms;
6447 fix.continuation = Some(cont);
6448 final_seat = resumed_seat;
6449 match resolved {
6450 Some(report) => {
6451 fix.addressed = report.addressed;
6452 fix.rejected = report.rejected;
6453 fix.notes =
6454 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6455 }
6456 None => fix.failed = failure,
6457 }
6458 }
6459 }
6460 }
6461 AgentOutcome::Dropped(o) => {
6462 fix.duration_ms = o.duration_ms;
6463 let why = o
6464 .dropped
6465 .as_ref()
6466 .map(|d| d.why.as_str())
6467 .unwrap_or("the CLI ended the stream without delivering its answer");
6468 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6469 }
6470 AgentOutcome::Quota(o) => {
6471 self.state.quota.push(QuotaLoss::from_report(
6472 final_seat.key.clone(),
6473 "fix",
6474 o.quota.as_ref(),
6475 ));
6476 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6477 }
6478 AgentOutcome::Failed(e) => fix.failed = Some(e),
6479 AgentOutcome::ResourceBlocked(e) => {
6480 fix.failed = Some(format!("the fixer was not started: {e}"));
6481 }
6482 }
6483 if fix.continuation.is_none() {
6486 fix.continuation = Some(ContinuationRecord::not_needed());
6487 }
6488 self.state.seats.insert(final_seat.key.clone(), final_seat);
6489
6490 let rescue_message = format!(
6491 "magi: operator-selected fix ({}) (uncommitted work)",
6492 self.state.operator_fixes[request_index]
6493 .findings
6494 .iter()
6495 .map(|f| f.id.as_str())
6496 .collect::<Vec<_>>()
6497 .join(", ")
6498 );
6499 let repo = self.state.repo.clone();
6500 let coordination = self.coordinate(&repo).await?;
6501 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6502 self.state.note_withheld("fix", &r.withheld);
6503 }
6504 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6505 fix.committed = after != head_at_request;
6506 git::worktree_remove(&self.state.repo, &fix_worktree)
6507 .await
6508 .ok();
6509 drop(coordination);
6510
6511 self.state.event(
6512 "fix",
6513 match &fix.failed {
6514 Some(reason) => format!(
6515 "operator fix: adoption report was lost ({reason}); {}",
6516 if fix.committed {
6517 "committed"
6518 } else {
6519 "NO new commit"
6520 }
6521 ),
6522 None => format!(
6523 "operator fix: {} addressed, {} rejected, {}",
6524 fix.addressed.len(),
6525 fix.rejected.len(),
6526 if fix.committed {
6527 "committed"
6528 } else {
6529 "NO new commit"
6530 }
6531 ),
6532 },
6533 );
6534
6535 for f in &mut self.state.operator_fixes[request_index].findings {
6542 f.outcome = if fix.failed.is_some() {
6543 OperatorFixOutcome::Unreported
6544 } else if fix.addressed.contains(&f.id) {
6545 OperatorFixOutcome::Addressed
6546 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6547 OperatorFixOutcome::Rejected { why: r.why.clone() }
6548 } else {
6549 OperatorFixOutcome::Unreported
6550 };
6551 }
6552
6553 let committed = fix.committed;
6554 if committed {
6555 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6556 }
6557 self.state.operator_fixes[request_index].fix = Some(fix);
6558 self.state.save()?;
6561 if self.state.resource_block.is_some() {
6562 return self.stop_for_resource();
6563 }
6564
6565 if committed {
6566 self.state.event(
6567 "fix",
6568 format!(
6569 "operator fix committed {}; opening a follow-up review-only run",
6570 short(&after)
6571 ),
6572 );
6573 let origin =
6576 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6577 match Self::review(
6578 &self.state.repo,
6579 &winner.branch,
6580 self.state.config.clone(),
6581 origin,
6582 )
6583 .await
6584 {
6585 Ok(mut follow_up) => {
6586 follow_up.state.event(
6587 "start",
6588 format!(
6589 "requested by an operator fix on run {} for finding(s) {}",
6590 self.state.id,
6591 self.state.operator_fixes[request_index]
6592 .findings
6593 .iter()
6594 .map(|f| f.id.as_str())
6595 .collect::<Vec<_>>()
6596 .join(", "),
6597 ),
6598 );
6599 follow_up.state.save()?;
6600 let follow_up_id = follow_up.state.id.clone();
6601 let adopted = match crate::direct::adopt(
6604 &follow_up.state,
6605 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6606 ) {
6607 Ok(a) => a,
6608 Err(e) => {
6609 self.state.event(
6612 "fix",
6613 format!(
6614 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6615 ),
6616 );
6617 self.state.save()?;
6618 return Ok(());
6619 }
6620 };
6621 let executed = follow_up.execute().await;
6622 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6623 if let Some(a) = adopted {
6624 a.finish(&follow_up.state, executed);
6625 }
6626 if let Some(e) = failure {
6627 self.state.event(
6628 "fix",
6629 format!(
6630 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6631 ),
6632 );
6633 }
6634 self.state.operator_fixes[request_index].follow_up_review_run =
6635 Some(follow_up_id);
6636 }
6637 Err(e) => {
6638 self.state.event(
6639 "fix",
6640 format!("committed the fix but could not open a follow-up review: {e:#}"),
6641 );
6642 }
6643 }
6644 self.state.save()?;
6645 }
6646
6647 Ok(())
6648 }
6649
6650 async fn ask_fixer(
6670 &mut self,
6671 winner: &Candidate,
6672 node: &'static str,
6673 round: Option<usize>,
6674 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6675 ) -> (SeatJob, SeatState, AgentOutcome) {
6676 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6677 self.ask_fixer_among(attempts, node, round, None, build)
6678 .await
6679 }
6680
6681 async fn ask_fixer_among(
6686 &mut self,
6687 attempts: Vec<(AgentSpec, String)>,
6688 node: &'static str,
6689 round: Option<usize>,
6690 restore: Option<(&Path, &str)>,
6691 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6692 ) -> (SeatJob, SeatState, AgentOutcome) {
6693 let run_id = self.state.id.clone();
6694 let prompts = self.state.config.prompts.clone();
6695 let cache = self.state.config.cache_dir();
6696 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6697 let mut last = None;
6698 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6699 let seat = self.seat(&key, &spec.id);
6700 let mut job = build(spec.clone(), seat);
6701 if i > 0 {
6702 job.stem = format!("{}-{}", job.stem, spec.id);
6703 }
6704 let ctx = WaveCtx {
6705 carry_seats: false,
6706 pauses: Some((&self.pause, &self.interrupt)),
6707 run: &run_id,
6708 node,
6709 prompts: &prompts,
6710 cache: cache.as_deref(),
6711 round,
6712 };
6713 let (seat, out) =
6714 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6715 let advances = match &out {
6718 AgentOutcome::Ok(o) => agent::output_advances(o),
6719 AgentOutcome::ResourceBlocked(_) => false,
6720 _ => true,
6721 };
6722 if let Some(next) = ids.get(i + 1)
6723 && advances
6724 {
6725 self.state.seats.insert(seat.key.clone(), seat.clone());
6726 let class =
6727 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6728 let reason = match &out {
6729 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6730 other => fail_reason(other),
6731 };
6732 if let Some((wt, rev)) = restore
6736 && let Err(e) = self.graft_restore(wt, rev).await
6737 {
6738 let why =
6739 format!("could not restore the worktree before the next fixer: {e:#}");
6740 tracing::warn!("{why}");
6741 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6742 AgentOutcome::ResourceBlocked(why)
6743 } else {
6744 AgentOutcome::Failed(why)
6745 };
6746 last = Some((job, seat, out));
6747 break;
6748 }
6749 record_handover(
6750 &mut self.state,
6751 node,
6752 &seat.key,
6753 &spec.id,
6754 next,
6755 &class,
6756 &reason,
6757 );
6758 continue;
6759 }
6760 last = Some((job, seat, out));
6761 break;
6762 }
6763 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6764 if let AgentOutcome::Quota(output) = &out
6765 && node != "graft"
6766 {
6767 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6772 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6773 self.state.quota.push(loss);
6774 self.state.seats.insert(seat.key.clone(), seat.clone());
6775 self.state.status = RunStatus::Blocked;
6776 return (
6777 job,
6778 seat,
6779 AgentOutcome::ResourceBlocked(
6780 "fixer unavailable; restore agent availability and resume".into(),
6781 ),
6782 );
6783 }
6784 (job, seat, out)
6785 }
6786
6787 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6798 use crate::recovery::{Mode, Repair};
6799 if !self.state.reviews.is_empty() {
6800 return Ok(());
6801 }
6802 let Some(mut rec) = self.state.recovery.clone() else {
6803 return Ok(());
6804 };
6805 if rec.mode != Mode::VerifyRepair {
6806 return Ok(());
6807 }
6808 let wt = winner.worktree.clone();
6809 let mut head = git::rev_parse(&wt, "HEAD").await?;
6810 match rec.repair.clone() {
6811 Repair::Done { .. } => return Ok(()),
6812 Repair::WaitingForRestore { before } => {
6813 self.graft_restore(&wt, &before).await?;
6814 head = before;
6815 rec.repair = Repair::NotStarted;
6816 self.state.recovery = Some(rec.clone());
6817 self.state.save()?;
6818 }
6819 Repair::Started { before } => {
6820 if let Err(e) = self.graft_restore(&wt, &before).await {
6824 if e.is::<crate::repo_lock::ResourceBlocked>() {
6825 return Err(e);
6826 }
6827 return self
6828 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6829 .await;
6830 }
6831 rec.repair = Repair::Done {
6832 before: before.clone(),
6833 after: before,
6834 committed: false,
6835 failed: Some("interrupted; rolled back".to_owned()),
6836 };
6837 self.state
6838 .event("recovery", "the repair was interrupted and rolled back");
6839 self.state.recovery = Some(rec);
6840 self.state.save()?;
6841 return Ok(());
6842 }
6843 Repair::NotStarted => {}
6844 }
6845 if crate::recovery::recheck(&mut rec, &head) {
6846 self.state.event("recovery", rec.summary());
6847 self.state.recovery = Some(rec);
6848 self.state.save()?;
6849 return Ok(());
6850 }
6851 rec.repair = Repair::Started {
6852 before: head.clone(),
6853 };
6854 self.state.recovery = Some(rec.clone());
6855 self.state.save()?;
6856
6857 let language = self.state.config.graph.language.clone();
6858 let sessions = self.state.config.graph.sessions;
6859 let artifacts = agent::artifacts_dir(&self.state.dir());
6860 let fix_prompt = prompt::recovery_fix(
6861 &self.state.instruction,
6862 &rec,
6863 self.state.config.graph.review_rounds,
6864 &language,
6865 );
6866 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6867 self.state.event(
6868 "recovery",
6869 format!(
6870 "repair: the fixer starts from the failed verification on {}",
6871 short(&head)
6872 ),
6873 );
6874 let (_job, seat, out) = self
6875 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6876 prompt: fix_prompt.clone(),
6877 spec,
6878 seat,
6879 cwd: wt.clone(),
6880 timeout,
6881 allow_write: true,
6882 sessions,
6883 artifacts: artifacts.clone(),
6884 stem: "fix-recovery".to_owned(),
6885 handover: None,
6886 })
6887 .await;
6888 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6889 rec.repair = Repair::WaitingForRestore { before: head };
6890 self.state.recovery = Some(rec);
6891 return self.stop_for_resource();
6892 }
6893 let failed = match &out {
6894 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6895 AgentOutcome::Quota(o) => {
6896 self.state.quota.push(QuotaLoss::from_report(
6897 seat.key.clone(),
6898 "fix",
6899 o.quota.as_ref(),
6900 ));
6901 Some("rate limited (quota); the fixer could not run".to_owned())
6902 }
6903 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6904 AgentOutcome::Failed(e) => Some(e.clone()),
6905 };
6906 self.state.seats.insert(seat.key.clone(), seat);
6907 let coordination = self.coordinate(&wt).await?;
6908 let failed = match failed {
6909 Some(why) => {
6911 if let Err(e) = git::restore_to(&wt, &head).await {
6912 return self
6913 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6914 .await;
6915 }
6916 Some(why)
6917 }
6918 None => {
6919 if let Ok(r) = git::rescue_commit(
6920 &wt,
6921 "magi: recovery repair of the failed verification (uncommitted work)",
6922 )
6923 .await
6924 {
6925 self.state.note_withheld("fix", &r.withheld);
6926 }
6927 None
6928 }
6929 };
6930 let after = git::rev_parse(&wt, "HEAD").await?;
6931 drop(coordination);
6932 self.state.event(
6933 "recovery",
6934 match &failed {
6935 Some(why) => format!("repair: nothing kept ({why})"),
6936 None if after != head => {
6937 format!("repair: committed {}", short(&after))
6938 }
6939 None => "repair: the fixer made no commit".to_owned(),
6940 },
6941 );
6942 rec.repair = Repair::Done {
6943 before: head.clone(),
6944 committed: after != head,
6945 after,
6946 failed,
6947 };
6948 self.state.recovery = Some(rec);
6949 self.state.save()?;
6950 Ok(())
6951 }
6952
6953 async fn stop_recovery(
6956 &mut self,
6957 mut rec: crate::recovery::RecoveryRecord,
6958 head: &str,
6959 why: String,
6960 ) -> Result<()> {
6961 rec.repair = crate::recovery::Repair::Started {
6963 before: head.to_owned(),
6964 };
6965 self.state.recovery = Some(rec);
6966 self.state
6967 .event("recovery", format!("repair could not be undone: {why}"));
6968 self.state.status = RunStatus::Blocked;
6969 self.state.save()?;
6970 bail!("recovery repair could not be rolled back: {why}")
6971 }
6972
6973 async fn review_loop(&mut self) -> Result<()> {
6974 if self.hold_base_sync().await? {
6975 return Ok(());
6976 }
6977 loop {
6978 if self
6979 .state
6980 .base_sync
6981 .as_ref()
6982 .is_some_and(|s| s.conflict.is_some())
6983 {
6984 return Ok(());
6985 }
6986 if self.catchup_repairable() {
6987 let record = self.state.reviews.last().expect("repairable round").clone();
6988 let winner = self
6989 .state
6990 .winner()
6991 .cloned()
6992 .context("catch-up repair has no winner")?;
6993 if !self.fix_review_round(&winner, record).await? {
6994 return Ok(());
6995 }
6996 }
6997 self.review_loop_pass().await?;
6998 if !self.catchup_repairable() {
6999 return Ok(());
7000 }
7001 }
7002 }
7003
7004 fn catchup_repairable(&self) -> bool {
7007 self.state.reviews.len() < self.state.config.graph.review_rounds
7008 && self.state.reviews.last().is_some_and(|r| {
7009 !r.clean
7010 && r.blocking == 0
7011 && r.fix.is_none()
7012 && r.e2e_status() == E2eStatus::Failed
7013 })
7014 }
7015
7016 async fn review_loop_pass(&mut self) -> Result<()> {
7017 if self
7022 .state
7023 .base_sync
7024 .as_ref()
7025 .is_some_and(|s| s.conflict.is_some())
7026 {
7027 return Ok(());
7028 }
7029 let run_id = self.state.id.clone();
7034 let prompts = self.state.config.prompts.clone();
7035 let Some(winner) = self.state.winner().cloned() else {
7036 return Ok(());
7037 };
7038 let granted_review = self
7039 .state
7040 .repair_grants
7041 .last()
7042 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7043 .cloned();
7044 let fresh_review = granted_review
7045 .as_ref()
7046 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7047 let max_rounds = granted_review
7048 .as_ref()
7049 .map_or(self.state.config.graph.review_rounds, |g| {
7050 g.review_start.saturating_add(1)
7051 });
7052 if let Some(status) =
7062 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7063 {
7064 if status == RunStatus::Gating {
7067 self.record_contested_handoff();
7068 }
7069 self.state.status = status;
7070 self.state.save()?;
7071 return Ok(());
7072 }
7073 self.state.status = RunStatus::Reviewing;
7074 if !fresh_review
7084 && self.state.reviews.last().is_some_and(|r| {
7085 r.e2e_status() == E2eStatus::ResourceBlocked
7086 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7087 })
7088 {
7089 let shell = self.state.config.shell();
7090 return self
7091 .stop_unresolved_review(
7092 "the last round's own verification never resolved",
7093 &shell,
7094 &winner.worktree,
7095 )
7096 .await;
7097 }
7098
7099 if granted_review.is_none() {
7100 self.recovery_repair(&winner).await?;
7101 }
7102 if self.resource_halt()? {
7103 return Ok(());
7104 }
7105
7106 let repo = self.state.repo.clone();
7107 let root = self.state.worktree_root();
7108 let language = self.state.config.graph.language.clone();
7109 let sessions = self.state.config.graph.sessions;
7110 let artifacts = self.merge_repair_artifacts();
7111 let base = self.landing_base();
7112 let base_short = short(&base);
7113 let reviewers = self.roles.reviewers.clone();
7114 let shell = self.state.config.shell();
7115
7116 for round in (self.state.reviews.len() + 1)..=max_rounds {
7117 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7118 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7119 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7120 let prev_verification = self
7129 .state
7130 .reviews
7131 .last()
7132 .and_then(|r| r.verification_summary(&head));
7133 let prior_section = match (&self.state.recovery, round) {
7136 (Some(rec), 1) if self.state.reviews.is_empty() => {
7137 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7138 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7139 Some(crate::recovery::head_of(
7140 &git::diff(&winner.worktree, reviewed, "HEAD")
7141 .await
7142 .unwrap_or_default(),
7143 crate::recovery::SINCE_DIFF_MAX,
7144 ))
7145 }
7146 _ => None,
7147 };
7148 Some(prompt::recovery_review(rec, since.as_deref()))
7149 }
7150 _ => None,
7151 };
7152
7153 let mut jobs = Vec::new();
7157 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7158 self.coordinate(&repo).await?
7159 } else {
7160 None
7161 };
7162 for (r, spec) in reviewers.iter().cloned().enumerate() {
7163 let wt = root.join(format!("review-{}", r + 1));
7164 if wt.exists() {
7167 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7168 } else {
7169 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7170 None
7171 } else {
7172 self.coordinate(&repo).await?
7173 };
7174 git::worktree_add_detached(&repo, &wt, &head).await?;
7175 }
7176 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7177 None
7178 } else {
7179 self.coordinate(&repo).await?
7180 };
7181 self.setup_seat_worktree(&repo, &wt, false).await?;
7182 let seat_key = format!("review-{}", r + 1);
7183 let spec = pick_start_spec(
7186 &self.roles.reviewer_roster,
7187 spec,
7188 self.state.seat_history.get(&seat_key),
7189 );
7190 let seat = self.seat(&seat_key, &spec.id);
7191 jobs.push(SeatJob {
7192 prompt: prompt::review(&prompt::ReviewCtx {
7193 instruction: &self.state.instruction,
7194 branch: &winner.branch,
7195 base_short: &base_short,
7196 stat: &stat,
7197 patch: &patch,
7198 verification: prev_verification.as_ref(),
7199 prior: prior_section.as_deref(),
7200 reviewers: reviewers.len(),
7201 round,
7202 rounds: max_rounds,
7203 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7206 lens: Lens::for_seat(r),
7207 language: &language,
7208 github_language: &self.state.config.graph.github_language,
7209 }),
7210 spec,
7211 seat,
7212 cwd: wt,
7213 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7214 allow_write: false,
7215 sessions,
7216 artifacts: artifacts.clone(),
7217 stem: format!("review-{round}-{}", r + 1),
7218 handover: None,
7219 });
7220 }
7221
7222 drop(coordination);
7223 self.state.event(
7224 "review",
7225 format!(
7226 "round {round}: {} reviewers on {}",
7227 jobs.len(),
7228 short(&head)
7229 ),
7230 );
7231 let mut quota_losses = Vec::new();
7232 let review_retries = self.state.config.graph.retries;
7233 let review_cache = self.state.config.cache_dir();
7234 let ctx = WaveCtx {
7235 carry_seats: true,
7236 pauses: Some((&self.pause, &self.interrupt)),
7237 run: &run_id,
7238 node: "review",
7239 prompts: &prompts,
7240 cache: review_cache.as_deref(),
7241 round: Some(round),
7242 };
7243 let results = ask_json_wave::<Review>(
7244 jobs,
7245 Arc::clone(&self.sem),
7246 review_retries,
7247 &self.roles.reviewer_roster,
7248 &ctx,
7249 &mut quota_losses,
7250 &mut self.state,
7251 &|_: &Review| Ok(()),
7252 )
7253 .await;
7254 if self.state.resource_block.is_some() {
7255 return self.stop_for_resource();
7256 }
7257 let round_quota_missing = quota_losses.len();
7261 self.state.quota.extend(quota_losses);
7262
7263 let mut records = Vec::new();
7264 let mut all_findings = Vec::new();
7265 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7266 let agent_id = seat.agent.clone();
7267 self.state.seats.insert(seat.key.clone(), seat);
7268 let mut record = ReviewRecord {
7269 reviewer: r + 1,
7270 agent: agent_id,
7271 summary: String::new(),
7272 findings: Vec::new(),
7273 vote: None,
7274 failed: None,
7275 duration_ms: 0,
7276 attempts,
7282 };
7283 match res {
7284 Ok((review, out)) => {
7285 record.summary =
7293 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7294 record.vote = Some(review.vote);
7295 record.duration_ms = out.duration_ms;
7296 for (n, mut f) in review.findings.into_iter().enumerate() {
7297 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7300 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7301 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7302 f.file = f
7308 .file
7309 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7310 all_findings.push(f.clone());
7311 record.findings.push(f);
7312 }
7313 self.state.event(
7314 "review",
7315 format!(
7316 "round {round}: reviewer {} voted {} with {} finding(s)",
7317 r + 1,
7318 review.vote.label(),
7319 record.findings.len()
7320 ),
7321 );
7322 }
7323 Err(e) => {
7324 record.failed = Some(e.to_string());
7325 self.state.event(
7326 "review",
7327 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7328 );
7329 }
7330 }
7331 records.push(record);
7332 }
7333
7334 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7341 let vote_split =
7342 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7343 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7344 if vote_split {
7345 self.state.event(
7346 "review",
7347 format!(
7348 "round {round}: votes split ({}) — one round of reconsideration",
7349 initial_votes
7350 .iter()
7351 .map(|v| v.label())
7352 .collect::<Vec<_>>()
7353 .join(", ")
7354 ),
7355 );
7356 let panel: Vec<ReviewSeatReport<'_>> = records
7359 .iter()
7360 .filter_map(|r| {
7361 r.vote.map(|vote| ReviewSeatReport {
7362 reviewer: r.reviewer,
7363 vote,
7364 summary: &r.summary,
7365 findings: &r.findings,
7366 })
7367 })
7368 .collect();
7369
7370 let mut jobs = Vec::new();
7371 let mut seats_at = Vec::new();
7372 for (r, spec) in reviewers.iter().cloned().enumerate() {
7373 if records[r].vote.is_none() {
7377 continue;
7378 }
7379 let wt = root.join(format!("review-{}", r + 1));
7380 let seat_key = format!("review-{}", r + 1);
7381 let spec = self.occupant(&seat_key, spec);
7382 let seat = self.seat(&seat_key, &spec.id);
7383 let build = |with_patch: bool| {
7391 prompt::review_reconsider(&ReviewReconsiderCtx {
7392 instruction: &self.state.instruction,
7393 reviewer: r + 1,
7394 lens: Lens::for_seat(r),
7395 panel: &panel,
7396 patch: with_patch.then_some(ReviewPatch {
7397 branch: &winner.branch,
7398 base_short: &base_short,
7399 stat: &stat,
7400 patch: &patch,
7401 }),
7402 round,
7403 rounds: max_rounds,
7404 language: &language,
7405 })
7406 };
7407 let full = build(true);
7408 let prompt = if has_context(&spec, &seat, sessions) {
7409 build(false)
7410 } else {
7411 full.clone()
7412 };
7413 jobs.push(SeatJob {
7414 prompt,
7415 spec,
7416 seat,
7417 cwd: wt,
7418 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7419 allow_write: false,
7420 sessions,
7421 artifacts: artifacts.clone(),
7422 stem: format!("review-{round}-reconsider-{}", r + 1),
7423 handover: Some(full),
7424 });
7425 seats_at.push(r);
7426 }
7427
7428 let mut recon_quota_losses = Vec::new();
7429 let recon_cache = self.state.config.cache_dir();
7430 let recon_ctx = WaveCtx {
7431 carry_seats: true,
7432 pauses: Some((&self.pause, &self.interrupt)),
7433 run: &run_id,
7434 node: "review",
7435 prompts: &prompts,
7436 cache: recon_cache.as_deref(),
7437 round: Some(round),
7438 };
7439 let recon_results = ask_json_wave::<ReviewRevote>(
7440 jobs,
7441 Arc::clone(&self.sem),
7442 review_retries,
7443 &self.roles.reviewer_roster,
7444 &recon_ctx,
7445 &mut recon_quota_losses,
7446 &mut self.state,
7447 &|_: &ReviewRevote| Ok(()),
7448 )
7449 .await;
7450 if self.state.resource_block.is_some() {
7451 return self.stop_for_resource();
7452 }
7453 self.state.quota.extend(recon_quota_losses);
7454
7455 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7456 let agent_id = seat.agent.clone();
7457 self.state.seats.insert(seat.key.clone(), seat);
7458 let mut rec = ReviewRevoteRecord {
7459 reviewer: r + 1,
7460 agent: agent_id,
7461 vote: None,
7462 reason: String::new(),
7463 failed: None,
7464 };
7465 match res {
7466 Ok((rv, _)) => {
7467 rec.vote = Some(rv.vote);
7468 rec.reason =
7469 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7470 self.state.event(
7471 "review",
7472 format!(
7473 "round {round}: reviewer {} revoted {}",
7474 r + 1,
7475 rv.vote.label()
7476 ),
7477 );
7478 }
7479 Err(e) => {
7480 rec.failed = Some(e.to_string());
7481 self.state.event(
7482 "review",
7483 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7484 );
7485 }
7486 }
7487 reconsideration.push(rec);
7488 }
7489 } else if initial_votes.len() > 1 {
7490 self.state.event(
7491 "review",
7492 format!(
7493 "round {round}: votes agreed ({}) — no reconsideration",
7494 initial_votes[0].label()
7495 ),
7496 );
7497 }
7498
7499 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7500 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7501 let defer_e2e =
7512 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7513 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7514 let reason =
7515 format!("{blocking} blocking finding(s) already required a fix this round");
7516 self.state.event(
7517 "verify",
7518 format!(
7519 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7520 {}); it will run once a round has none left",
7521 short(&head)
7522 ),
7523 );
7524 (Vec::new(), false, true, Some(reason))
7525 } else {
7526 let e2e_commands = self.state.config.verify.e2e.clone();
7527 let cache_dir = self.state.config.cache_dir();
7528 let context = format!("round {round}");
7529 let wait = self.repository_wait();
7530 let (e2e, verify_retried) = with_cache_lease(
7531 &mut self.state,
7532 cache_dir.as_deref(),
7533 "e2e",
7534 "e2e",
7535 &winner.worktree,
7536 &head,
7537 verify_timeout,
7538 &context,
7539 &wait,
7540 |state, budget| {
7541 let shell = shell.clone();
7542 let e2e_commands = e2e_commands.clone();
7543 let worktree = winner.worktree.clone();
7544 let context = context.clone();
7545 async move {
7546 run_e2e_with_retry(
7547 state,
7548 &shell,
7549 &e2e_commands,
7550 &worktree,
7551 budget,
7552 &context,
7553 )
7554 .await
7555 }
7556 },
7557 )
7558 .await?;
7559 (e2e, verify_retried, false, None)
7560 };
7561
7562 let expected = records.len();
7563 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7564 let incomplete = answered < expected;
7565 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7566 let policy = self.state.config.graph.incomplete_review;
7567 let clean = round_is_clean(
7568 blocking,
7569 e2e_ok,
7570 answered,
7571 expected,
7572 round_quota_missing,
7573 policy,
7574 );
7575
7576 let mut round_record = ReviewRound {
7577 round,
7578 head: head.clone(),
7579 verified_head: None,
7580 verified_at: None,
7581 reviews: records,
7582 e2e,
7583 verify_retried,
7584 e2e_deferred,
7585 e2e_defer_reason,
7586 fix: None,
7587 blocking,
7588 answered,
7589 expected,
7590 clean,
7591 progressed: false,
7592 vote_split,
7593 reconsideration,
7594 verdict: None,
7595 };
7596 round_record.verdict = ReviewVote::worst(
7600 round_record
7601 .final_votes()
7602 .into_iter()
7603 .map(|(_, _, vote)| vote),
7604 );
7605 if !matches!(
7616 round_record.e2e_status(),
7617 E2eStatus::Deferred | E2eStatus::NotConfigured
7618 ) {
7619 round_record.verified_head = Some(head.clone());
7620 round_record.verified_at = Some(Timestamp::now());
7621 }
7622
7623 if incomplete {
7624 let missing: Vec<String> = round_record
7625 .reviews
7626 .iter()
7627 .filter(|r| r.failed.is_some())
7628 .map(|r| format!("review-{}", r.reviewer))
7629 .collect();
7630 self.state.event(
7631 "review",
7632 format!(
7633 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7634 missing.join(", ")
7635 ),
7636 );
7637 }
7638
7639 if clean {
7640 self.state.event(
7641 "review",
7642 if incomplete && policy == IncompleteReviewPolicy::Warn {
7643 format!(
7644 "round {round}: clean (warn policy, incomplete panel) — no \
7645 blocking findings from the seats that answered, verification green"
7646 )
7647 } else if incomplete {
7648 format!(
7649 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7650 quorum) — no blocking findings from the seats that answered, \
7651 verification green",
7652 expected - answered
7653 )
7654 } else {
7655 format!("round {round}: clean — no blocking findings, verification green")
7656 },
7657 );
7658 self.state.reviews.push(round_record);
7659 self.state.status = RunStatus::Gating;
7660 self.state.save()?;
7661 return Ok(());
7662 }
7663
7664 if incomplete && blocking == 0 && e2e_ok {
7672 self.state.reviews.push(round_record);
7673 self.state.save()?;
7674 if round == max_rounds {
7675 self.state.status = RunStatus::Blocked;
7676 self.state.event(
7677 "review",
7678 format!(
7679 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7680 refusing to call it clean",
7681 expected - answered
7682 ),
7683 );
7684 return Ok(());
7685 }
7686 continue;
7687 }
7688
7689 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7701 self.state.reviews.push(round_record);
7702 return self
7703 .stop_unresolved_review(
7704 "the round's own verification could not run",
7705 &shell,
7706 &winner.worktree,
7707 )
7708 .await;
7709 }
7710
7711 if round == max_rounds {
7712 self.state.reviews.push(round_record);
7713 return self
7714 .stop_reviewing(
7715 &format!(
7716 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7717 ),
7718 &shell,
7719 &winner.worktree,
7720 )
7721 .await;
7722 }
7723
7724 if !self.fix_review_round(&winner, round_record).await? {
7725 return Ok(());
7726 }
7727 }
7728 Ok(())
7729 }
7730
7731 async fn fix_review_round(
7734 &mut self,
7735 winner: &Candidate,
7736 mut round_record: ReviewRound,
7737 ) -> Result<bool> {
7738 let round = round_record.round;
7739 let shell = self.state.config.shell();
7740 let step = self.run_fix_round(winner, &mut round_record).await?;
7741 if matches!(step, FixStep::Resource) {
7742 self.stop_for_resource()?;
7743 return Ok(false);
7744 }
7745 if self.state.reviews.len() < round {
7746 self.state.reviews.push(round_record);
7747 } else {
7748 self.state.reviews[round - 1] = round_record;
7749 }
7750 self.state.save()?;
7751
7752 if matches!(step, FixStep::Recorded { report_lost: true }) {
7765 self.stop_reviewing(
7766 "the fixer's adoption report never came back, even after resuming its \
7767 own seat; refusing to start another round against the same worktree \
7768 while that is unresolved",
7769 &shell,
7770 &winner.worktree,
7771 )
7772 .await?;
7773 return Ok(false);
7774 }
7775
7776 let streak = self
7777 .state
7778 .reviews
7779 .iter()
7780 .rev()
7781 .take_while(|r| !r.progressed)
7782 .count();
7783 if streak >= STAGNANT_LIMIT {
7784 self.stop_reviewing(
7785 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7786 &shell,
7787 &winner.worktree,
7788 )
7789 .await?;
7790 return Ok(false);
7791 }
7792 Ok(true)
7793 }
7794
7795 async fn run_fix_round(
7797 &mut self,
7798 winner: &Candidate,
7799 round_record: &mut ReviewRound,
7800 ) -> Result<FixStep> {
7801 let round = round_record.round;
7802 let max_rounds = self.state.config.graph.review_rounds;
7803 let run_id = self.state.id.clone();
7804 let prompts = self.state.config.prompts.clone();
7805 let language = self.state.config.graph.language.clone();
7806 let sessions = self.state.config.graph.sessions;
7807 let artifacts = self.merge_repair_artifacts();
7808 let base = self.landing_base();
7809 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7810 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7811 let this_round_verification = round_record.verification_summary(&head);
7812 let all_findings: Vec<_> = round_record
7813 .reviews
7814 .iter()
7815 .flat_map(|r| r.findings.iter().cloned())
7816 .collect();
7817 let blocking_findings: Vec<_> = all_findings
7820 .iter()
7821 .filter(|f| f.severity.blocks())
7822 .cloned()
7823 .collect();
7824 let fix_prompt = prompt::fix_in(
7825 &self.state.instruction,
7826 &blocking_findings,
7827 this_round_verification.as_ref(),
7828 round,
7829 max_rounds,
7830 &language,
7831 &self.state.config.graph.github_language,
7832 );
7833 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7834 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7835 let (job, seat, out) = self
7836 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7837 prompt: fix_prompt.clone(),
7838 spec,
7839 seat,
7840 cwd: winner.worktree.clone(),
7841 timeout,
7842 allow_write: true,
7843 sessions,
7844 artifacts: artifacts.clone(),
7845 stem: format!("fix-{round}"),
7846 handover: None,
7847 })
7848 .await;
7849 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7852 return Ok(FixStep::Resource);
7853 }
7854 let agent_id = seat.agent.clone();
7855
7856 let mut fix = FixRecord {
7857 agent: agent_id,
7858 addressed: Vec::new(),
7859 rejected: Vec::new(),
7860 notes: String::new(),
7861 committed: false,
7862 failed: None,
7863 duration_ms: 0,
7864 continuation: None,
7865 };
7866 let mut continuation = ContinuationRecord::not_needed();
7867 let mut final_seat = seat.clone();
7868 match out {
7869 AgentOutcome::Ok(o) => {
7870 fix.duration_ms = o.duration_ms;
7871 let parsed = verdict::extract_json::<FixReport>(&o.text);
7872 let incomplete_reason = match &parsed {
7879 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7880 "the reply parsed, but it reported a command whose own CLI never \
7881 confirmed an exit status"
7882 .to_owned(),
7883 ),
7884 Ok(_) => None,
7885 Err(e) => Some(e.to_string()),
7886 };
7887 match incomplete_reason {
7888 None => {
7889 let report = parsed.expect("checked Ok above");
7890 fix.addressed = report.addressed;
7891 fix.rejected = report.rejected;
7892 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7893 }
7894 Some(reason) => {
7895 let (resumed_seat, resolved, failure, cont) = self
7896 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7897 .await;
7898 fix.duration_ms += cont.cumulative_wait_ms;
7899 continuation = cont;
7900 final_seat = resumed_seat;
7901 match resolved {
7902 Some(report) => {
7903 fix.addressed = report.addressed;
7904 fix.rejected = report.rejected;
7905 fix.notes =
7906 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7907 }
7908 None => fix.failed = failure,
7909 }
7910 }
7911 }
7912 }
7913 AgentOutcome::Dropped(o) => {
7915 fix.duration_ms = o.duration_ms;
7916 let why = o
7917 .dropped
7918 .as_ref()
7919 .map(|d| d.why.as_str())
7920 .unwrap_or("the CLI ended the stream without delivering its answer");
7921 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7922 }
7923 AgentOutcome::Quota(o) => {
7924 self.state.quota.push(QuotaLoss::from_report(
7925 final_seat.key.clone(),
7926 "fix",
7927 o.quota.as_ref(),
7928 ));
7929 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7930 }
7931 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7932 }
7933 if self.state.resource_block.is_some() {
7934 self.state.seats.insert(final_seat.key.clone(), final_seat);
7935 return Ok(FixStep::Resource);
7936 }
7937 fix.continuation = Some(continuation);
7938 self.state.seats.insert(final_seat.key.clone(), final_seat);
7939 let coordination = self.coordinate(&winner.worktree).await?;
7940 if let Ok(r) = git::rescue_commit(
7941 &winner.worktree,
7942 &format!("magi: review round {round} fixes (uncommitted work)"),
7943 )
7944 .await
7945 {
7946 self.state.note_withheld("fix", &r.withheld);
7947 }
7948 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7949 fix.committed = after != before;
7950 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7958 drop(coordination);
7959 let progressed = diff_after != patch;
7960 let commit_note = if fix.committed {
7961 "committed"
7962 } else {
7963 "NO new commit"
7964 };
7965 let tree_note = if progressed {
7966 "changed vs base"
7967 } else {
7968 "unchanged vs base"
7969 };
7970 self.state.event(
7971 "fix",
7972 match &fix.failed {
7973 Some(reason) => {
7979 format!(
7980 "round {round}: fixer's adoption report was lost ({reason}); \
7981 {commit_note}, tree {tree_note}"
7982 )
7983 }
7984 None => format!(
7985 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7986 {tree_note}{}",
7987 fix.addressed.len(),
7988 fix.rejected.len(),
7989 if continuation.outcome == ContinuationOutcome::Resumed {
7990 format!(
7991 " (adoption report recovered after {} continuation(s))",
7992 continuation.attempts
7993 )
7994 } else {
7995 String::new()
7996 },
7997 ),
7998 },
7999 );
8000 round_record.fix = Some(fix);
8001 round_record.progressed = progressed;
8002 Ok(FixStep::Recorded {
8003 report_lost: matches!(
8004 continuation.outcome,
8005 ContinuationOutcome::Exhausted
8006 | ContinuationOutcome::QuotaLost
8007 | ContinuationOutcome::NoSession
8008 ),
8009 })
8010 }
8011
8012 fn record_contested_handoff(&mut self) {
8046 if self.state.contested_handoff.is_some() {
8047 return;
8048 }
8049 let Some(contested) = self
8050 .state
8051 .reviews
8052 .last()
8053 .and_then(ReviewRound::contested_handoff)
8054 else {
8055 return;
8056 };
8057 self.state.event(
8058 "review",
8059 format!(
8060 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8061 wait for the owner's approval",
8062 contested.findings.len(),
8063 contested.rejecters.len()
8064 ),
8065 );
8066 self.state.contested_handoff = Some(contested);
8067 }
8068
8069 async fn stop_unresolved_review(
8070 &mut self,
8071 why: &str,
8072 shell: &[String],
8073 worktree: &Path,
8074 ) -> Result<()> {
8075 self.stop_reviewing_inner(why, shell, worktree, true).await
8076 }
8077
8078 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8079 self.stop_reviewing_inner(why, shell, worktree, false).await
8080 }
8081
8082 async fn stop_reviewing_inner(
8083 &mut self,
8084 why: &str,
8085 shell: &[String],
8086 worktree: &Path,
8087 repairable: bool,
8088 ) -> Result<()> {
8089 let round_idx = self.state.reviews.len() - 1;
8090 let needs_catchup_run = matches!(
8098 self.state.reviews[round_idx].e2e_status(),
8099 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8100 );
8101 if needs_catchup_run {
8102 let round = self.state.reviews[round_idx].round;
8103 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8104 let commands = self.state.config.verify.e2e.clone();
8105 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8106 let cache_dir = self.state.config.cache_dir();
8107 let context = format!(
8108 "round {round}: verification unresolved, catching up before the final decision"
8109 );
8110 let wait = self.repository_wait();
8111 let (outcomes, verify_retried) = with_cache_lease(
8112 &mut self.state,
8113 cache_dir.as_deref(),
8114 "e2e",
8115 "e2e",
8116 worktree,
8117 &attempted_head,
8118 timeout,
8119 &context,
8120 &wait,
8121 |state, budget| {
8122 let shell = shell.to_vec();
8123 let commands = commands.clone();
8124 let context = context.clone();
8125 async move {
8126 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8127 .await
8128 }
8129 },
8130 )
8131 .await?;
8132 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8133 self.state.record_base_progress(&attempted_head);
8134 }
8135 let last = &mut self.state.reviews[round_idx];
8136 last.e2e = outcomes;
8137 last.verify_retried = verify_retried;
8138 last.verified_head = Some(attempted_head);
8145 last.verified_at = Some(Timestamp::now());
8146 if verify_inconclusive(&last.e2e) {
8147 self.state.save()?;
8154 return Ok(());
8155 }
8156 last.e2e_deferred = false;
8157 if last.answered == last.expected {
8162 last.clean = round_is_clean(
8163 last.blocking,
8164 last.e2e.iter().all(CommandOutcome::ok),
8165 last.answered,
8166 last.expected,
8167 0,
8168 self.state.config.graph.incomplete_review,
8169 );
8170 }
8171 }
8172 let last = &self.state.reviews[round_idx];
8173 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8174
8175 match last.e2e_status() {
8176 E2eStatus::Failed => {
8177 let red: Vec<String> = last
8178 .e2e
8179 .iter()
8180 .filter(|o| !o.ok())
8181 .map(|o| {
8182 format!(
8183 "`{}` -> {:?}\n{}",
8184 o.command,
8185 o.code,
8186 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8187 )
8188 })
8189 .collect();
8190 let can_repair = last.blocking == 0 && last.fix.is_none();
8191 self.state.reviews[round_idx].clean = false;
8192 self.state
8193 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8194 self.state.status = RunStatus::Blocked;
8195 if repairable
8196 && can_repair
8197 && self.state.reviews.len() < self.state.config.graph.review_rounds
8198 {
8199 self.state.status = RunStatus::Reviewing;
8200 }
8201 }
8202 E2eStatus::ResourceBlocked => {
8207 self.state.event(
8208 "review",
8209 format!(
8210 "{why}; e2e could not run (resource unavailable); not \
8211 deciding yet"
8212 ),
8213 );
8214 }
8215 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8216 self.state.event(
8217 "review",
8218 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8219 );
8220 self.record_contested_handoff();
8221 self.state.status = RunStatus::Gating;
8222 }
8223 }
8224 self.state.save()?;
8225 Ok(())
8226 }
8227
8228 async fn gate(&mut self) -> Result<()> {
8231 if self.hold_base_sync().await? {
8232 return Ok(());
8233 }
8234 if self.state.status == RunStatus::Failed
8246 || self
8247 .state
8248 .base_sync
8249 .as_ref()
8250 .is_some_and(|s| s.conflict.is_some())
8251 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8252 != Some(RunStatus::Gating)
8253 {
8254 return Ok(());
8255 }
8256 let Some(winner) = self.state.winner().cloned() else {
8257 return Ok(());
8258 };
8259 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8260 self.state.reconcile_verification_head(&head);
8261 self.state.save()?;
8262 let wait = self.repository_wait();
8263 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8264 return Ok(());
8265 }
8266 if self.state.gate_ran {
8267 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8278 self.state.status = RunStatus::Blocked;
8279 self.state.save()?;
8280 }
8281 return Ok(());
8282 }
8283 self.state.status = RunStatus::Gating;
8284 let mut outcomes = self.run_gate(&winner).await?;
8285 let mut attempted_head = self.state.gate_head.clone();
8286 loop {
8287 if verify_inconclusive(&outcomes) {
8298 self.state.save()?;
8299 return Ok(());
8300 }
8301 if outcomes.iter().all(CommandOutcome::ok) {
8302 break;
8303 }
8304 let fixes_before = self.state.gate_fixes.len();
8305 match self.gate_fix_round(&winner, &outcomes).await? {
8306 GateFix::Retry => {
8307 outcomes = self.run_gate(&winner).await?;
8308 attempted_head = self.state.gate_head.clone();
8309 }
8310 GateFix::Stop => {
8311 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8312 if self.state.gate_fixes.len() > fixes_before
8313 && attempted_head.as_deref() != Some(current.as_str())
8314 && verify_current_head(&mut self.state, &winner, &wait).await?
8315 {
8316 outcomes = self.run_gate(&winner).await?;
8319 attempted_head = self.state.gate_head.clone();
8320 } else {
8321 break;
8322 }
8323 }
8324 GateFix::Defer => {
8325 self.state.save()?;
8326 return Ok(());
8327 }
8328 }
8329 }
8330 let passed = outcomes.iter().all(CommandOutcome::ok);
8331 self.state.gate = outcomes;
8332 self.state.gate_ran = true;
8333 self.state.gate_head = attempted_head;
8334 if !self.state.gate.is_empty()
8335 && !verify_inconclusive(&self.state.gate)
8336 && let Some(head) = self.state.gate_head.clone()
8337 {
8338 self.state.record_base_progress(&head);
8339 }
8340 self.state.save()?;
8341 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8342 return Ok(());
8343 }
8344 if !passed {
8345 self.state.status = RunStatus::Blocked;
8346 let spent = self.state.gate_fixes.len();
8347 self.state.event(
8348 "gate",
8349 if spent == 0 {
8350 "gate failed; not merging".to_owned()
8351 } else {
8352 format!("gate failed after {spent} gate-fix round(s); not merging")
8353 },
8354 );
8355 }
8356 self.state.save()?;
8357 Ok(())
8358 }
8359
8360 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8370 let commands = state.config.verify.pre_gate.clone();
8371 if commands.is_empty() {
8372 return;
8373 }
8374 let shell = state.config.shell();
8375 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8376 let (outcomes, _) = run_commands(
8377 state,
8378 "pre_gate",
8379 "pre_gate",
8380 0,
8381 &shell,
8382 &commands,
8383 &winner.worktree,
8384 timeout,
8385 )
8386 .await;
8387 for o in &outcomes {
8388 if !o.ok() {
8389 tracing::warn!(
8390 "pre_gate `{}` failed ({:?}); the gate decides",
8391 o.command,
8392 o.code
8393 );
8394 }
8395 state.event(
8396 "pre_gate",
8397 format!(
8398 "`{}` -> {}",
8399 o.command,
8400 if o.ok() {
8401 "pass".to_owned()
8402 } else {
8403 format!(
8404 "FAIL ({:?})\n{}",
8405 o.code,
8406 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8407 )
8408 }
8409 ),
8410 );
8411 }
8412 state.pre_gate = outcomes;
8413 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8414 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8415 Ok(head) => {
8416 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8417 state.pre_gate_commit = Some(head);
8418 }
8419 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8420 },
8421 Ok(false) => {}
8422 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8423 }
8424 if let Err(e) = state.save() {
8425 tracing::warn!("could not persist the pre_gate record: {e:#}");
8426 }
8427 }
8428
8429 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8432 let wait = self.repository_wait();
8433 run_landing_gate(&mut self.state, winner, &wait).await
8434 }
8435
8436 async fn gate_fix_round(
8449 &mut self,
8450 winner: &Candidate,
8451 outcomes: &[CommandOutcome],
8452 ) -> Result<GateFix> {
8453 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8454 g.phase == crate::repair_budget::Phase::Fixing
8455 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8456 }) {
8457 self.state.gate_fixes.len().saturating_add(1)
8458 } else {
8459 self.state.config.graph.gate_fix_rounds
8460 };
8461 let spent = self.state.gate_fixes.len();
8462 if spent >= cap {
8463 if cap > 0 {
8464 self.state.event(
8465 "gate",
8466 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8467 );
8468 }
8469 return Ok(GateFix::Stop);
8470 }
8471 if !gate_fixable(outcomes) {
8472 self.state.event(
8473 "gate",
8474 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8475 command or similar); not spending a fix round on it",
8476 );
8477 return Ok(GateFix::Stop);
8478 }
8479 let min_free = self.state.config.disk.min_free_bytes;
8480 if min_free > 0 {
8481 match crate::disk::free_bytes(&winner.worktree) {
8482 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8483 Ok(free) => {
8484 self.state.event(
8485 "gate",
8486 format!(
8487 "only {free} bytes free ({min_free} required by `[disk] \
8488 min_free_bytes`); not spending a fix round on a failure the disk \
8489 may explain"
8490 ),
8491 );
8492 return Ok(GateFix::Stop);
8493 }
8494 Err(e) => {
8495 self.state.event(
8496 "gate",
8497 format!("free disk space could not be measured ({e:#}); no fix round"),
8498 );
8499 return Ok(GateFix::Stop);
8500 }
8501 }
8502 }
8503
8504 let attempt = spent + 1;
8505 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8506 let base = self.landing_base();
8507 let fix_prompt = prompt::gate_fix_in(
8508 &self.state.instruction,
8509 &failed,
8510 attempt,
8511 cap,
8512 &self.state.config.graph.language,
8513 &self.state.config.graph.github_language,
8514 );
8515 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8516 let sessions = self.state.config.graph.sessions;
8517 let artifacts = self.merge_repair_artifacts();
8518 self.state.event(
8519 "gate",
8520 format!("gate failed; gate-fix round {attempt} of {cap}"),
8521 );
8522 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8523 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8524 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8527 head: before.clone(),
8528 passed: false,
8529 outcomes: Vec::new(),
8530 });
8531 self.state.save()?;
8532 let (_, seat, out) = self
8533 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8534 prompt: fix_prompt.clone(),
8535 spec,
8536 seat,
8537 cwd: winner.worktree.clone(),
8538 timeout,
8539 allow_write: true,
8540 sessions,
8541 artifacts: artifacts.clone(),
8542 stem: format!("gate-fix-{attempt}"),
8543 handover: None,
8544 })
8545 .await;
8546 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8549 return Ok(GateFix::Defer);
8550 }
8551 let mut record = GateFixRecord {
8552 agent: seat.agent.clone(),
8553 failed,
8554 notes: String::new(),
8555 committed: false,
8556 error: None,
8557 };
8558 match out {
8559 AgentOutcome::Ok(o) => {
8560 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8563 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8564 }
8565 }
8566 AgentOutcome::Dropped(_) => {
8567 record.error = Some("the CLI dropped the stream".to_owned());
8568 }
8569 AgentOutcome::Quota(o) => {
8570 self.state.quota.push(QuotaLoss::from_report(
8571 seat.key.clone(),
8572 "gate-fix",
8573 o.quota.as_ref(),
8574 ));
8575 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8576 }
8577 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8578 }
8579 self.state.seats.insert(seat.key.clone(), seat);
8580 let coordination = self.coordinate(&winner.worktree).await?;
8581 if let Ok(r) = git::rescue_commit(
8582 &winner.worktree,
8583 &format!("magi: gate fix {attempt} (uncommitted work)"),
8584 )
8585 .await
8586 {
8587 self.state.note_withheld("gate-fix", &r.withheld);
8588 }
8589 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8590 record.committed = after != before;
8591 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8592 let note = record.error.clone();
8593 self.state.gate_fixes.push(record);
8594 self.state.save()?;
8595 if !changed {
8596 self.state.event(
8597 "gate",
8598 match note {
8599 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8600 None => format!("gate-fix round {attempt}: the tree did not change"),
8601 },
8602 );
8603 return Ok(GateFix::Stop);
8604 }
8605 self.state.event(
8606 "gate",
8607 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8608 );
8609
8610 drop(coordination);
8611 let wait = self.repository_wait();
8612 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8613 }
8614
8615 async fn guarded_pr_message(
8622 &mut self,
8623 winner: &Candidate,
8624 facts: Option<&BranchFacts>,
8625 posting: bool,
8626 ) -> Result<Option<PrMessage>> {
8627 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8628 let head = match final_head.as_deref() {
8629 Some(h) => crate::pr_outcome::FinalHead::At(h),
8630 None => crate::pr_outcome::FinalHead::Unknown,
8631 };
8632 let summarized = if posting {
8634 Some(self.summarize_change(winner, final_head.as_deref()).await)
8635 } else {
8636 None
8637 };
8638 if self.state.resource_block.is_some() {
8643 self.state.event(
8644 "pr-summary",
8645 "a shared resource was busy; nothing was published, resume to summarize again",
8646 );
8647 return Ok(None);
8648 }
8649 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8650 anyhow::bail!(
8651 "the final commit moved during public description generation; retry before publishing"
8652 );
8653 }
8654 let note = match (&summarized, &self.state.change_summary) {
8655 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8656 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8657 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8658 (None, _) => ChangeNote::NotAsked,
8659 };
8660 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8661 if !posting {
8662 let identity = crate::scrub::Identity::current();
8663 return Ok(Some(PrMessage {
8664 title: crate::scrub::scrub(&pr.title, &identity),
8665 body: crate::scrub::scrub(&pr.body, &identity),
8666 }));
8667 }
8668 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8671 if let Some(known) = self
8672 .state
8673 .github_text
8674 .clone()
8675 .filter(|g| g.fingerprint == raw_fp)
8676 {
8677 return self.settle_withheld(winner, &pr, known).await;
8678 }
8679 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8680 let mut violations =
8681 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8682 let identity = crate::scrub::Identity::current();
8683 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8684 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8685 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8686 {
8687 violations.push(crate::github_text::Violation::SensitiveData);
8688 }
8689 let has_language = violations
8693 .iter()
8694 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8695 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8696 self.state.event(
8697 "github-text",
8698 if has_language {
8699 format!("description rejected: {violations:?}; requesting one rewrite")
8700 } else {
8701 format!("description rejected: {violations:?}; asking the owner")
8702 },
8703 );
8704 let mut rewritten = false;
8705 let mut rejected_rewrite = None;
8706 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8707 {
8708 let output = text;
8709 if let Some(title) = summary_title(&output) {
8710 let raw_body = summary_without_title(&output);
8713 let clean_title = crate::scrub::scrub(&title, &identity);
8714 let mut body = crate::scrub::scrub(&raw_body, &identity);
8715 let redacted = clean_title != title || body != raw_body;
8716 let title = clean_title;
8717 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8718 if !complete {
8719 self.state.event(
8720 "github-text",
8721 "description rewrite omitted required findings",
8722 );
8723 }
8724 if !body.trim().is_empty() {
8725 body.push_str("\n\n");
8726 body.push_str(&public_pr_evidence(&self.state, head));
8727 }
8728 let marker = format!("magi:run/{}", self.state.id);
8729 if !body.trim().is_empty() && !body.contains(&marker) {
8730 body.push_str(&format!("\n\n{marker}"));
8731 }
8732 let rewrite = PrMessage {
8733 title: title.clone(),
8734 body: body.clone(),
8735 };
8736 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8737 let mut failed = crate::github_text::check_for(
8738 &self.state.config.graph,
8739 &title,
8740 &body,
8741 redecision,
8742 );
8743 if (!crate::github_text::shareable(&title)
8744 || !crate::github_text::shareable(&body))
8745 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8746 {
8747 failed.push(crate::github_text::Violation::SensitiveData);
8748 }
8749 if complete
8750 && !title.trim().is_empty()
8751 && !body.trim().is_empty()
8752 && !failed.is_empty()
8753 {
8754 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8755 title: title.clone(),
8756 body: body.clone(),
8757 failed: failed
8758 .iter()
8759 .map(|v| crate::github_text::category(*v).to_owned())
8760 .collect(),
8761 });
8762 }
8763 if complete
8764 && !title.trim().is_empty()
8765 && !raw_body.trim().is_empty()
8766 && failed.is_empty()
8767 {
8768 decision = redecision;
8769 pr = PrMessage { title, body };
8770 rewritten = true;
8771 self.state
8772 .event("github-text", "description rewrite passed");
8773 if redacted {
8774 self.state.event(
8775 "github-text",
8776 "sensitive data removed from description rewrite",
8777 );
8778 }
8779 }
8780 }
8781 }
8782 if !rewritten {
8783 self.state.event(
8784 "github-text",
8785 "description rewrite unavailable or rejected; asking the owner",
8786 );
8787 if let Some(w) =
8788 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8789 {
8790 self.retire_github_text_question("the withheld text changed");
8793 let record = crate::run::GithubTextAsk {
8794 rewrite: rejected_rewrite,
8795 fingerprint: raw_fp,
8796 title: w.title,
8797 body: w.body,
8798 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8799 question: None,
8800 asks: 0,
8801 resolved: false,
8802 chosen_title: None,
8803 choice: None,
8804 };
8805 return self.settle_withheld(winner, &pr, record).await;
8806 }
8807 }
8808 }
8809 let (title, body) =
8810 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8811 Ok(Some(PrMessage { title, body }))
8812 }
8813
8814 async fn summarize_change(
8825 &mut self,
8826 winner: &Candidate,
8827 head: Option<&str>,
8828 ) -> std::result::Result<(), String> {
8829 let Some(head) = head else {
8830 return Err("the final commit could not be read".to_owned());
8831 };
8832 let remote = self.state.config.merge.remote.clone();
8833 let tracking = format!("{remote}/{}", self.state.base_branch);
8834 if !git::rev_exists(&winner.worktree, &tracking).await {
8835 return Err(format!("the remote base `{tracking}` could not be read"));
8836 }
8837 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8838 Ok(b) if !b.is_empty() => b,
8839 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8840 };
8841 let evidence = crate::pr_outcome::summary_evidence(
8842 &self.state,
8843 crate::pr_outcome::FinalHead::At(head),
8844 );
8845 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8846 if self
8847 .state
8848 .change_summary
8849 .as_ref()
8850 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8851 {
8852 return Ok(());
8853 }
8854 if self.state.change_summary.take().is_some() {
8855 self.state.event(
8856 "pr-summary",
8857 "the final diff or evidence changed; the earlier change summary was discarded",
8858 );
8859 }
8860 if let Err(e) = write_artifact(
8862 &self.state,
8863 &format!("public-description-{fingerprint}.md"),
8864 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8865 ) {
8866 tracing::warn!("could not save public description evidence: {e:#}");
8867 }
8868 let chain = match agent::pick_chain(
8869 &self.state.config.agents,
8870 self.state.config.roles.synthesizer.as_ref(),
8871 &agent::installed,
8872 "synthesizer",
8873 ) {
8874 Ok(chain) => chain,
8875 Err(e) => {
8876 tracing::warn!("no agent for the change summary: {e:#}");
8877 self.state.event("pr-summary", "no summary agent available");
8878 return Err("no agent was available to summarize it".to_owned());
8879 }
8880 };
8881 let subjects = git::subjects(&winner.worktree, &base, head)
8882 .await
8883 .unwrap_or_default();
8884 let stat = git::diff_stat(&winner.worktree, &base, head)
8885 .await
8886 .unwrap_or_default();
8887 let stat: String = stat.chars().take(6_000).collect();
8888 let mut prompt = crate::prompt::change_summary_in(
8889 &base,
8890 head,
8891 &subjects,
8892 stat.trim(),
8893 &self.state.config.graph.github_language,
8894 );
8895 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8896 let sessions = self.state.config.graph.sessions;
8897 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8898 let prompts = self.state.config.prompts.clone();
8899 let cache = self.state.config.cache_dir();
8900 let run = self.state.id.clone();
8901 for (i, spec) in chain.into_iter().enumerate() {
8902 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8903 let github_spec = match agent::for_github_text(
8904 &spec,
8905 &self.state.config.graph.github_language,
8906 &winner.worktree,
8907 ) {
8908 Ok(spec) => spec,
8909 Err(e) => {
8910 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8911 continue;
8912 }
8913 };
8914 let job = SeatJob {
8915 spec: github_spec,
8916 seat,
8917 cwd: winner.worktree.clone(),
8918 prompt: prompt.clone(),
8919 timeout,
8920 allow_write: false,
8921 sessions,
8922 artifacts: agent::artifacts_dir(&self.state.dir()),
8923 stem: if i == 0 {
8924 "change-summary".to_owned()
8925 } else {
8926 format!("change-summary-{}", spec.id)
8927 },
8928 handover: None,
8929 };
8930 let ctx = WaveCtx {
8931 run: &run,
8932 node: "pr-summary",
8933 prompts: &prompts,
8934 cache: cache.as_deref(),
8935 round: None,
8936 carry_seats: false,
8937 pauses: Some((&self.pause, &self.interrupt)),
8938 };
8939 let (seat, outcome) =
8940 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8941 self.state.seats.insert(seat.key.clone(), seat);
8942 let text = match outcome {
8943 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8944 AgentOutcome::ResourceBlocked(why) => {
8945 if self.state.resource_block.is_none() {
8948 self.state.resource_block = Some(crate::run::ResourceBlock {
8949 cause: crate::run::ResourceBlockCause::Other,
8950 node: "pr-summary".to_owned(),
8951 kind: "shared resource".to_owned(),
8952 detail: why,
8953 at: Timestamp::now(),
8954 });
8955 }
8956 return Err("a shared resource was busy".to_owned());
8957 }
8958 _ => continue,
8959 };
8960 let blind = &self.state.config.blind;
8961 let Some(title) = summary_title(&text) else {
8962 continue;
8963 };
8964 let body: String = summary_without_title(&text)
8965 .lines()
8966 .filter(|l| !l.contains("magi:run/"))
8967 .collect::<Vec<_>>()
8968 .join("\n");
8969 let body = blind::sanitize_prose(body.trim(), blind);
8970 let title = blind::sanitize_prose(&title, blind);
8971 if body.trim().is_empty() || title.trim().is_empty() {
8972 continue;
8973 }
8974 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8975 continue;
8976 }
8977 if git::rev_parse(&winner.worktree, "HEAD")
8979 .await
8980 .ok()
8981 .as_deref()
8982 != Some(head)
8983 {
8984 return Err("the branch moved while the summary was written".to_owned());
8985 }
8986 if git::merge_base(&winner.worktree, &tracking, head)
8987 .await
8988 .ok()
8989 .as_deref()
8990 != Some(base.as_str())
8991 || crate::pr_outcome::summary_evidence(
8992 &self.state,
8993 crate::pr_outcome::FinalHead::At(head),
8994 ) != evidence
8995 {
8996 return Err("the summary evidence changed while it was written".to_owned());
8997 }
8998 self.state.change_summary = Some(crate::run::ChangeSummary {
8999 head: head.to_owned(),
9000 base,
9001 evidence: fingerprint,
9002 title,
9003 body,
9004 });
9005 self.state.event(
9006 "pr-summary",
9007 format!("change summary written by {} for the final diff", spec.id),
9008 );
9009 if let Err(e) = self.state.save() {
9010 tracing::warn!("could not save the change summary: {e:#}");
9011 }
9012 return Ok(());
9013 }
9014 self.state
9015 .event("pr-summary", "no agent produced a change summary");
9016 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9017 }
9018
9019 async fn rewrite_pr_text(
9029 &mut self,
9030 winner: &Candidate,
9031 pr: &PrMessage,
9032 violations: &[crate::github_text::Violation],
9033 ) -> Option<String> {
9034 let prompt = format!(
9035 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9036 pr.title, pr.body
9037 );
9038 let sessions = self.state.config.graph.sessions;
9039 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9040 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9043 let key = format!("impl-{}", winner.label);
9044 if let Some(spec) = self
9045 .state
9046 .config
9047 .agents
9048 .iter()
9049 .find(|a| a.id == winner.agent)
9050 .cloned()
9051 && let Some(existing) = self.state.seats.get(&key)
9052 && existing.agent == spec.id
9053 && agent::has_session(spec.kind, existing, sessions)
9054 {
9055 attempts.push((spec, Some(existing.clone())));
9056 }
9057 match agent::pick_chain(
9058 &self.state.config.agents,
9059 self.state.config.roles.language_judge.as_ref(),
9060 &agent::installed,
9061 "language_judge",
9062 ) {
9063 Ok(chain) => {
9064 for spec in chain {
9065 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9066 continue;
9067 }
9068 attempts.push((spec, None));
9069 }
9070 }
9071 Err(e) if attempts.is_empty() => {
9072 tracing::warn!("no agent for the description rewrite: {e:#}");
9073 self.state
9074 .event("github-text", "no rewrite agent available");
9075 return None;
9076 }
9077 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9078 }
9079 let prompts = self.state.config.prompts.clone();
9080 let cache = self.state.config.cache_dir();
9081 let run = self.state.id.clone();
9082 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9083 let resumed = resumed_seat.is_some();
9084 let seat = resumed_seat.unwrap_or_else(|| {
9085 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9086 });
9087 let github_spec = match agent::for_github_text(
9088 &spec,
9089 &self.state.config.graph.github_language,
9090 &winner.worktree,
9091 ) {
9092 Ok(spec) => spec,
9093 Err(e) => {
9094 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9095 continue;
9096 }
9097 };
9098 let job = SeatJob {
9099 spec: github_spec,
9100 seat,
9101 cwd: winner.worktree.clone(),
9102 prompt: prompt.clone(),
9103 timeout: retry_budget(timeout_full, resumed),
9104 allow_write: false,
9105 sessions,
9106 artifacts: agent::artifacts_dir(&self.state.dir()),
9107 stem: if i == 0 {
9108 "github-text-rewrite".to_owned()
9109 } else {
9110 format!("github-text-rewrite-{}", spec.id)
9111 },
9112 handover: None,
9113 };
9114 let ctx = WaveCtx {
9115 run: &run,
9116 node: "github-text",
9117 prompts: &prompts,
9118 cache: cache.as_deref(),
9119 round: None,
9120 carry_seats: false,
9121 pauses: Some((&self.pause, &self.interrupt)),
9122 };
9123 let (seat, outcome) =
9124 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9125 self.state.seats.insert(seat.key.clone(), seat);
9126 match outcome {
9127 AgentOutcome::Ok(o)
9128 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9129 {
9130 return Some(o.text);
9131 }
9132 AgentOutcome::ResourceBlocked(_) => return None,
9133 _ => {}
9134 }
9135 }
9136 None
9137 }
9138
9139 fn retire_github_text_question(&mut self, why: &str) {
9141 let Some(id) = self
9142 .state
9143 .github_text
9144 .as_ref()
9145 .and_then(|g| g.question.clone())
9146 else {
9147 return;
9148 };
9149 let _ = ask::Questions::open().update(&id, |q| {
9150 q.abandon(why);
9151 Ok(())
9152 });
9153 }
9154
9155 async fn settle_withheld(
9159 &mut self,
9160 winner: &Candidate,
9161 pr: &PrMessage,
9162 mut g: crate::run::GithubTextAsk,
9163 ) -> Result<Option<PrMessage>> {
9164 use crate::github_text as gt;
9165 let store = ask::Questions::open();
9166 let timeout = self.state.config.graph.answer_timeout;
9167 if !g.resolved {
9168 let mut standing = g
9169 .question
9170 .as_ref()
9171 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9172 if let Some(q) = standing.as_ref()
9173 && q.status.open()
9174 && gt::expired(q, timeout)
9175 {
9176 let why = format!("no answer within {}s of asking", timeout.max(1));
9177 standing = store
9178 .update(&q.id, |q| {
9179 q.abandon(&why);
9180 Ok(())
9181 })
9182 .ok()
9183 .map(|(q, ())| q);
9184 }
9185 let mut ask_again: Option<Option<Vec<String>>> = None;
9186 match standing {
9187 None if g.asks <= 1 => {
9190 g.asks = 0;
9191 ask_again = Some(None);
9192 }
9193 None => {
9194 g.resolved = true;
9195 g.chosen_title = None;
9196 g.choice = Some(GithubTextChoice::Fallback);
9197 }
9198 Some(q) if q.status.open() => {
9199 self.park_on_github_text(&g)?;
9200 return Ok(None);
9201 }
9202 Some(q) => match gt::read_reply(&q) {
9203 gt::Reply::Fallback => {
9204 g.resolved = true;
9205 g.chosen_title = None;
9206 g.choice = Some(GithubTextChoice::Fallback);
9207 }
9208 gt::Reply::Original => {
9209 g.resolved = true;
9210 g.chosen_title = None;
9211 g.choice = Some(GithubTextChoice::Original);
9212 }
9213 gt::Reply::Rewrite => {
9214 g.resolved = true;
9215 g.chosen_title = None;
9216 g.choice = Some(if g.rewrite.is_some() {
9217 GithubTextChoice::Rewrite
9218 } else {
9219 GithubTextChoice::Fallback
9220 });
9221 }
9222 gt::Reply::Title(text) => {
9223 let first = text
9224 .lines()
9225 .map(str::trim)
9226 .find(|l| !l.is_empty())
9227 .unwrap_or("");
9228 let decision = if first.is_empty() {
9229 None
9230 } else {
9231 let candidate = PrMessage {
9232 title: first.to_owned(),
9233 body: String::new(),
9234 };
9235 self.judge_pr_language(&winner.worktree, &candidate).await
9236 };
9237 match gt::vet_title(
9238 &text,
9239 decision,
9240 self.state.config.graph.github_english(),
9241 ) {
9242 Ok(title) => {
9243 g.resolved = true;
9244 g.chosen_title = Some(title);
9245 g.choice = Some(GithubTextChoice::MyTitle);
9246 }
9247 Err(cats) if g.asks < 2 => {
9248 ask_again = Some(Some(
9249 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9250 ))
9251 }
9252 Err(_) => {
9253 g.resolved = true;
9254 g.chosen_title = None;
9255 g.choice = Some(GithubTextChoice::Fallback);
9256 }
9257 }
9258 }
9259 },
9260 }
9261 if let Some(rejected) = ask_again {
9262 let w = gt::Withheld {
9265 title: g.title,
9266 body: g.body,
9267 categories: g.categories.clone(),
9268 };
9269 let artifact = match crate::run::write_artifact(
9272 &self.state,
9273 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9274 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9275 ) {
9276 Ok(p) => Some(p),
9277 Err(e) => {
9278 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9279 None
9280 }
9281 };
9282 let (detail, comparison) = gt::question_content(
9283 &w,
9284 &pr.title,
9285 &pr.body,
9286 &self.state.id,
9287 rejected.as_deref(),
9288 artifact.as_deref(),
9289 g.rewrite.as_ref(),
9290 );
9291 let mut q = ask::Question::new(
9292 self.state.id.clone(),
9293 gt::ASK_NODE.to_owned(),
9294 gt::ASK_SEAT.to_owned(),
9295 gt::question_summary(&self.state.config.graph.language, &w),
9296 detail,
9297 gt::question_choices(&w, g.rewrite.as_ref()),
9298 );
9299 q.posting_comparison = Some(comparison);
9300 q.answer_timeout = timeout;
9301 g.question = Some(q.id.clone());
9304 g.asks += 1;
9305 self.state.github_text = Some(g.clone());
9306 self.state.event(
9307 "github-text",
9308 format!("asking the owner about the withheld text ({})", q.short()),
9309 );
9310 self.state.save()?;
9311 store
9312 .put_executable(&mut q)
9313 .context("file the posting-gate question")?;
9314 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9315 tracing::warn!(
9316 "could not notify about posting-gate question {}: {e:#}",
9317 q.short()
9318 );
9319 }
9320 self.park_on_github_text(&g)?;
9321 return Ok(None);
9322 }
9323 }
9324 let resolved = g.chosen_title.clone();
9325 let original = g.choice == Some(GithubTextChoice::Original);
9326 self.state.event(
9328 "github-text",
9329 if original {
9330 format!(
9331 "posting the original text for: {}",
9332 match (g.title, g.body) {
9333 (true, true) => "title, description",
9334 (true, false) => "title",
9335 _ => "description",
9336 }
9337 )
9338 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9339 "posting the saved rewrite title and description".to_owned()
9340 } else if resolved.is_some() {
9341 "posting the owner's replacement title".to_owned()
9342 } else {
9343 "posting the neutral text for the withheld fields".to_owned()
9344 },
9345 );
9346 self.state.github_text = Some(g.clone());
9347 self.state.save()?;
9348 if g.choice == Some(GithubTextChoice::Rewrite)
9349 && let Some(r) = g.rewrite.as_ref()
9350 {
9351 return Ok(Some(PrMessage {
9352 title: r.title.clone(),
9353 body: r.body.clone(),
9354 }));
9355 }
9356 let w = gt::Withheld {
9357 title: g.title,
9358 body: g.body,
9359 categories: g.categories.clone(),
9360 };
9361 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9362 if let Some(t) = resolved {
9363 title = t;
9364 }
9365 let english = gt::LanguageDecision {
9367 title_english: true,
9368 body_english: true,
9369 body_complete: true,
9370 };
9371 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9372 if original {
9373 if g.title {
9376 title = pr.title.clone();
9377 }
9378 if g.body {
9379 body = pr.body.clone();
9380 }
9381 }
9382 Ok(Some(PrMessage { title, body }))
9383 }
9384
9385 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9386 self.state.github_text = Some(g.clone());
9387 self.state.end_pass_parked(ParkCause::Question);
9388 self.state.parked = true;
9389 self.state.event(
9390 "github-text",
9391 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9392 );
9393 self.state.save()
9394 }
9395
9396 async fn judge_pr_language(
9399 &mut self,
9400 cwd: &Path,
9401 pr: &PrMessage,
9402 ) -> Option<crate::github_text::LanguageDecision> {
9403 self.state.config.roles.language_judge.as_ref()?;
9404 if !self.state.config.graph.language_gate() {
9405 return None;
9406 }
9407 let decision =
9408 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9409 self.state.event(
9410 "github-text",
9411 if decision.is_some() {
9412 "language decided by the language judge"
9413 } else {
9414 "language judge unavailable; using built-in heuristics"
9415 },
9416 );
9417 decision
9418 }
9419
9420 async fn merge(&mut self) -> Result<()> {
9421 if self.resource_halt()? || self.hold_base_sync().await? {
9422 return Ok(());
9423 }
9424 if self
9439 .state
9440 .base_sync
9441 .as_ref()
9442 .is_some_and(|s| s.conflict.is_some())
9443 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9444 != Some(RunStatus::Gating)
9445 || !self.state.gate_allows_progress()
9453 {
9454 return Ok(());
9455 }
9456 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9465 return Ok(());
9466 }
9467 let Some(winner) = self.state.winner().cloned() else {
9468 return Ok(());
9469 };
9470 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9471 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9472 self.state.status = RunStatus::Blocked;
9473 self.state.event(
9474 "gate",
9475 "winner changed after gate; fresh verification required",
9476 );
9477 self.state.save()?;
9478 return Ok(());
9479 }
9480 let wait = self.repository_wait();
9481 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9482 return Ok(());
9483 }
9484 let repo = self.state.repo.clone();
9485 let base = self.state.base_branch.clone();
9486 let mode = self.state.config.merge.mode;
9487 let style = self.state.config.merge.style;
9488 let facts = if is_review_run(&self.state) {
9489 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9490 let start = review_base(
9491 &repo,
9492 &self.state.config.merge.remote,
9493 &base,
9494 &self.state.base_commit,
9495 &winner.branch,
9496 )
9497 .await;
9498 branch_facts(&repo, &start, &winner.branch).await
9499 } else {
9500 None
9501 };
9502 let leaked = if is_review_run(&self.state) {
9505 leaked_subjects(&self.state, &winner.branch).await
9506 } else {
9507 Some(Vec::new())
9508 };
9509 let Some(pr) = self
9513 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9514 .await?
9515 else {
9516 return Ok(());
9517 };
9518 let message = pr.commit_message();
9519
9520 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9521 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9522 return Ok(());
9523 }
9524 let outcome = match mode {
9525 MergeMode::None => MergeOutcome {
9526 mode,
9527 ok: true,
9528 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9529 empty: false,
9530 },
9531 MergeMode::Pr | MergeMode::Local
9532 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9533 {
9534 MergeOutcome {
9535 mode,
9536 ok: false,
9537 detail: empty_candidate_detail(&self.state, &base),
9538 empty: true,
9539 }
9540 }
9541 MergeMode::Local => {
9542 let on = git::current_branch(&repo).await?;
9543 if on.as_deref() != Some(base.as_str()) {
9544 MergeOutcome {
9545 mode,
9546 ok: false,
9547 detail: format!(
9548 "{} has {} checked out, not the base branch {base}",
9549 repo.display(),
9550 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9551 ),
9552 empty: false,
9553 }
9554 } else if !git::is_clean(&repo).await? {
9555 MergeOutcome {
9556 mode,
9557 ok: false,
9558 detail: format!("{} is dirty; refusing to merge", repo.display()),
9559 empty: false,
9560 }
9561 } else {
9562 let out = match style {
9563 MergeStyle::Merge => {
9564 git::merge_no_ff(&repo, &winner.branch, &message).await?
9565 }
9566 MergeStyle::Squash => {
9567 git::merge_squash(&repo, &winner.branch, &message).await?
9568 }
9569 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9570 };
9571 MergeOutcome {
9572 mode,
9573 ok: out.ok(),
9574 detail: if out.ok() { out.stdout } else { out.stderr },
9575 empty: false,
9576 }
9577 }
9578 }
9579 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9580 self.state.pr_publication_pending = true;
9581 self.state.save()?;
9582 let out =
9583 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9584 self.state.pr_publication_pending = out.is_err();
9585 MergeOutcome {
9586 mode,
9587 ok: out.is_ok(),
9588 detail: match out {
9589 Ok(url) => url,
9590 Err(e) => format!("{e:#}"),
9591 },
9592 empty: false,
9593 }
9594 }
9595 MergeMode::Pr => {
9596 let remote = self.state.config.merge.remote.clone();
9597 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9598 let pushed = if self.state.foreign_checkout.is_some() {
9599 let refspec = format!("{delivery_head}:refs/heads/{public}");
9600 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9601 } else {
9602 git::push(&winner.worktree, &remote, &winner.branch).await?
9603 };
9604 if !pushed.ok() {
9605 MergeOutcome {
9606 mode,
9607 ok: false,
9608 detail: pushed.stderr,
9609 empty: false,
9610 }
9611 } else {
9612 if self.state.foreign_checkout.is_some() {
9613 self.state.pr_pushed_head = Some(delivery_head.clone());
9614 }
9615 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9621 let out = match pr_merge_plan(found) {
9622 PrPlan::Create => {
9623 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9624 .await
9625 }
9626 PrPlan::Adopt { url, title } => {
9627 self.state
9628 .event("merge", format!("Pr: adopted open pull request {url}"));
9629 if self.state.github_text.as_ref().is_some_and(|g| {
9630 g.resolved
9631 && g.choice == Some(GithubTextChoice::Rewrite)
9632 && g.rewrite.is_some()
9633 }) {
9634 land::set_pr_text_verbatim(
9637 &winner.worktree,
9638 &url,
9639 &pr.title,
9640 &pr.body,
9641 )
9642 .await
9643 .map(|()| url)
9644 } else {
9645 if title != pr.title
9646 && (!is_review_run(&self.state)
9647 || leaked
9648 .as_deref()
9649 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9650 && let Err(e) =
9651 if self.state.github_text.as_ref().is_some_and(|g| {
9652 g.resolved
9653 && g.title
9654 && g.choice == Some(GithubTextChoice::Original)
9655 }) {
9656 land::set_pr_title_verbatim(
9658 &winner.worktree,
9659 &url,
9660 &pr.title,
9661 )
9662 .await
9663 } else {
9664 land::set_pr_title(
9665 &mut self.state,
9666 &winner.worktree,
9667 &url,
9668 &pr.title,
9669 )
9670 .await
9671 }
9672 {
9673 tracing::warn!("could not refresh title of {url}: {e:#}");
9674 self.state
9675 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9676 }
9677 Ok(url)
9678 }
9679 }
9680 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9681 };
9682 match out {
9683 Ok(url) => MergeOutcome {
9684 mode,
9685 ok: true,
9686 detail: url,
9687 empty: false,
9688 },
9689 Err(e) => MergeOutcome {
9690 mode,
9691 ok: false,
9692 detail: e.to_string(),
9693 empty: false,
9694 },
9695 }
9696 }
9697 }
9698 };
9699
9700 self.state.status = match (mode, outcome.ok) {
9701 (MergeMode::None, _) => RunStatus::Ready,
9702 (_, true) if outcome.empty => RunStatus::Ready,
9703 (MergeMode::Pr, true) => RunStatus::Ready,
9704 (_, true) => RunStatus::Merged,
9705 (_, false) => RunStatus::Blocked,
9706 };
9707 self.state.event(
9708 "merge",
9709 format!(
9710 "{:?}: {}",
9711 mode,
9712 outcome.detail.lines().next().unwrap_or("")
9713 ),
9714 );
9715 self.state.merge = Some(outcome);
9716 self.state.save()?;
9717
9718 if self.state.config.graph.land
9724 && mode == MergeMode::Pr
9725 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9726 {
9727 self.run_land().await?;
9728 }
9729 self.settle_questions();
9734 Ok(())
9735 }
9736
9737 async fn run_land(&mut self) -> Result<()> {
9748 crate::pr_head::ensure_owner(&self.state)?;
9749 let url = self
9750 .state
9751 .pr
9752 .as_ref()
9753 .map(|pr| pr.url.clone())
9754 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9755 .unwrap_or_default();
9756 let url = url.lines().next().unwrap_or("").trim().to_owned();
9757 if !url.starts_with("http") {
9758 return Ok(());
9759 }
9760 loop {
9763 match land::land(&mut self.state, &url).await {
9764 Ok(_)
9765 if self
9766 .state
9767 .merge_repairs
9768 .last()
9769 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9770 {
9771 if self.repair_merge_review().await? {
9772 continue;
9773 }
9774 }
9775 Ok(_) if self.state.resource_block.is_some() => {
9776 return self.stop_for_resource();
9777 }
9778 Ok(pr) if self.state.parked => {
9779 let _ = pr;
9783 }
9784 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9785 self.state.status = RunStatus::Superseded;
9786 self.state.save()?;
9787 }
9788 Ok(pr) => {
9789 self.state.status = match pr.state {
9790 land::PrLifecycle::Merged => RunStatus::Merged,
9791 _ => RunStatus::Blocked,
9792 };
9793 if bump::should_release_bump(self.state.status)
9800 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9801 {
9802 self.state
9808 .event("bump", format!("release bump skipped: {e:#}"));
9809 }
9810 if self.state.status == RunStatus::Merged {
9813 crate::followup::after_merge(&mut self.state, &pr.url).await;
9814 }
9815 self.state.save()?;
9816 }
9817 Err(e) => {
9818 if e.is::<crate::repo_lock::ResourceBlocked>() {
9819 return Err(e);
9820 }
9821 self.state.status = RunStatus::Blocked;
9822 self.state.event("land", format!("gave up: {e}"));
9823 self.state.save()?;
9824 }
9825 }
9826 break;
9827 }
9828 Ok(())
9829 }
9830
9831 async fn repair_merge_review(&mut self) -> Result<bool> {
9834 use crate::run::MergeRepairPhase as Phase;
9835 crate::pr_head::ensure_owner(&self.state)?;
9836 let i = self.state.merge_repairs.len() - 1;
9837 let request = self.state.merge_repairs[i].clone();
9838 if request.phase == Phase::Fixing {
9839 let restored = match self.state.winner().cloned() {
9840 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9841 None => Err(anyhow::anyhow!("repair has no winning branch")),
9842 };
9843 let why = match restored {
9844 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9845 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9846 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9847 };
9848 self.hold_merge_repair(i, why)?;
9849 return Ok(false);
9850 }
9851 if request.phase == Phase::Reviewing && self.state.gate_ran {
9856 let winner = self
9857 .state
9858 .winner()
9859 .context("repair has no winning branch")?;
9860 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9861 if self.state.gate_head.as_deref() != Some(head.as_str())
9862 || !git::is_clean(&winner.worktree).await?
9863 {
9864 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9865 return Ok(false);
9866 }
9867 }
9868 let failure = if i >= self.state.config.graph.land_rounds
9869 || self.state.config.graph.review_rounds == 0
9870 {
9871 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9872 } else {
9873 None
9874 };
9875 if let Some(why) = failure {
9876 self.hold_merge_repair(i, why)?;
9877 return Ok(false);
9878 }
9879 if request.phase == Phase::Requested {
9880 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9881 Ok(pr) => pr,
9882 Err(e) => {
9883 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9884 return Ok(false);
9885 }
9886 };
9887 self.state.merge_repairs[i].phase = Phase::Fixing;
9888 self.state.land_approval = None;
9889 self.state.land_armed_head = None;
9890 self.state.save()?;
9891 let evidence = serde_json::to_string_pretty(&request)?;
9894 let reason = format!(
9895 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9896 );
9897 let budget = self.state.config.graph.land_rounds;
9898 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9899 match result {
9900 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9901 Ok(land::Fixed::Deferred) => {
9902 let worktree = self
9906 .state
9907 .winner()
9908 .context("repair has no winning branch")?
9909 .worktree
9910 .clone();
9911 self.graft_restore(&worktree, &request.head).await?;
9912 self.state.merge_repairs[i].phase = Phase::Requested;
9913 self.stop_for_resource()?;
9914 return Ok(false);
9915 }
9916 Ok(land::Fixed::Failed(why)) => {
9917 self.hold_merge_repair(
9918 i,
9919 format!(
9920 "repair failed: {why}; inspect the branch and run an explicit review"
9921 ),
9922 )?;
9923 return Ok(false);
9924 }
9925 Err(e) => {
9926 self.hold_merge_repair(
9927 i,
9928 format!(
9929 "repair failed: {e:#}; inspect the branch and run an explicit review"
9930 ),
9931 )?;
9932 return Ok(false);
9933 }
9934 }
9935 self.begin_merge_repair_review(i)?;
9939 }
9940 self.review_loop().await?;
9941 self.gate().await?;
9942 if self.resource_halt()? || self.state.parked {
9943 return Ok(false);
9944 }
9945 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9946 != Some(RunStatus::Gating)
9947 || !self.state.gate_allows_progress()
9948 {
9949 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9950 return Ok(false);
9951 }
9952 let winner = self
9955 .state
9956 .winner()
9957 .cloned()
9958 .context("repair has no winning branch")?;
9959 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9960 if self.state.gate_head.as_deref() != Some(after.as_str())
9961 || !git::is_clean(&winner.worktree).await?
9962 {
9963 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9964 return Ok(false);
9965 }
9966 let expected = self
9967 .state
9968 .pr_pushed_head
9969 .as_deref()
9970 .unwrap_or(&request.head)
9971 .to_owned();
9972 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9973 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9976 return Ok(false);
9977 }
9978 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9979 let refspec = format!("{after}:refs/heads/{public}");
9980 let push = git::git_raw(
9981 &winner.worktree,
9982 &["push", &lease, &self.state.config.merge.remote, &refspec],
9983 )
9984 .await;
9985 match push {
9986 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9987 Ok(out) => {
9988 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9989 return Ok(false);
9990 }
9991 Err(e) => {
9992 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9993 return Ok(false);
9994 }
9995 }
9996 self.state.merge_repairs[i].phase = Phase::Complete;
9997 self.state.status = RunStatus::Landing;
9998 self.state.event(
9999 "repair",
10000 "new review and gate completed; fresh owner merge decision required",
10001 );
10002 self.state.save()?;
10003 Ok(true)
10004 }
10005
10006 fn merge_repair_artifacts(&self) -> PathBuf {
10007 let artifacts = agent::artifacts_dir(&self.state.dir());
10008 match self
10009 .state
10010 .merge_repairs
10011 .last()
10012 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10013 {
10014 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10015 None => artifacts,
10016 }
10017 }
10018
10019 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10020 if let Some(e2e) = self.state.head_e2e.take() {
10021 self.state
10022 .verification_history
10023 .push(crate::run::VerificationEvidence {
10024 superseded_by: None,
10025 archived_at: Some(Timestamp::now()),
10026 fixes_spent: Some(self.state.gate_fixes.len()),
10027 gate_head: None,
10028 gate_ran: false,
10029 gate: Vec::new(),
10030 e2e: Some(e2e),
10031 });
10032 }
10033 self.state.reviews.clear();
10034 self.state.gate.clear();
10035 self.state.gate_ran = false;
10036 self.state.gate_head = None;
10037 self.state.gate_fixes.clear();
10038 self.state.gate_repair_e2e = None;
10039 self.state.pre_gate.clear();
10040 self.state.pre_gate_commit = None;
10041 self.state.contested_handoff = None;
10042 self.state.land_reruns = None;
10043 self.state.land_approval = None;
10044 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10045 self.state.status = RunStatus::Reviewing;
10046 self.state.save()
10047 }
10048
10049 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10050 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10051 self.state.merge_repairs[index].failure = Some(why.clone());
10052 self.state.status = RunStatus::Blocked;
10053 self.state.event("repair", why);
10054 self.state.save()
10055 }
10056
10057 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10061 if let Some(existing) = self.state.seats.get(key)
10062 && existing.agent == agent
10063 {
10064 return existing.clone();
10065 }
10066 let fresh = if self.state.seats.contains_key(key) {
10069 handover_seat(key, agent, self.state.next_seat_seed())
10070 } else {
10071 SeatState::new(key, agent, self.state.seed)
10072 };
10073 self.state.seats.insert(key.to_owned(), fresh.clone());
10074 fresh
10075 }
10076
10077 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10083 match self.state.seats.get(key) {
10084 Some(s) if s.agent != spec.id => {
10085 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10086 }
10087 _ => spec,
10088 }
10089 }
10090
10091 fn view(&self, c: &Candidate) -> CandidateView {
10093 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10094 .unwrap_or_default();
10095 let (patch, _) = blind::sanitize_patch(
10096 &format!("candidate {} patch", c.label),
10097 &raw,
10098 &self.state.config.blind,
10099 );
10100 CandidateView {
10101 label: c.label,
10102 branch: c.branch.clone(),
10103 summary: c.summary.clone(),
10104 stat: c.stat.clone(),
10105 patch,
10106 }
10107 }
10108
10109 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10111 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10112 prompt::judge(
10113 "(see above)",
10114 &views,
10115 self.roles.judges.len(),
10116 base_short,
10117 "en",
10118 )
10119 }
10120
10121 fn vote_prompt_full(
10127 &self,
10128 j: usize,
10129 labels: &[char],
10130 language: &str,
10131 candidates: &[Candidate],
10132 base_short: &str,
10133 ) -> String {
10134 let mut text = format!(
10135 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10136 prompt::final_vote(labels, language),
10137 self.state.instruction,
10138 self.candidate_block(candidates, base_short)
10139 );
10140 if let Some(own) = self
10141 .state
10142 .judgements
10143 .get(j)
10144 .filter(|r| !r.ranking.is_empty())
10145 {
10146 let reasons = own
10147 .reasons
10148 .iter()
10149 .map(|(k, v)| format!("- {k}: {v}"))
10150 .collect::<Vec<_>>()
10151 .join("\n");
10152 text.push_str(&format!(
10153 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10154 own.ranking.iter().collect::<String>(),
10155 if reasons.is_empty() {
10156 ""
10157 } else {
10158 ", because:\n"
10159 }
10160 ));
10161 }
10162 if !self.state.deliberation.is_empty() {
10163 text.push_str("\n# What was argued before this vote\n");
10164 for t in self.transcript(&[], j) {
10165 text.push_str(&format!(
10166 "\n## {}{}\n\n{}\n",
10167 t.who,
10168 if t.is_self { " (you)" } else { "" },
10169 t.body.trim()
10170 ));
10171 }
10172 }
10173 text
10174 }
10175
10176 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10183 let mut turns = Vec::new();
10184 for j in &self.state.judgements {
10185 if j.ranking.is_empty() {
10186 continue;
10187 }
10188 let reasons = j
10189 .reasons
10190 .iter()
10191 .map(|(k, v)| format!("- {k}: {v}"))
10192 .collect::<Vec<_>>()
10193 .join("\n");
10194 turns.push(Turn {
10195 who: format!("Judge {} (opening ranking)", j.judge),
10196 is_self: j.judge == self_idx + 1,
10197 body: format!(
10198 "Ranked {}{}{reasons}",
10199 j.ranking.iter().collect::<String>(),
10200 if reasons.is_empty() {
10201 ""
10202 } else {
10203 ", because:\n"
10204 }
10205 ),
10206 });
10207 }
10208 for t in self
10209 .state
10210 .deliberation
10211 .iter()
10212 .flat_map(|r| r.turns.iter())
10213 .chain(current)
10214 {
10215 turns.push(Turn {
10216 who: format!("Judge {}", t.judge),
10217 is_self: t.judge == self_idx + 1,
10218 body: t.body.clone(),
10219 });
10220 }
10221 turns
10222 }
10223}
10224
10225fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10227 agent::has_session(spec.kind, seat, sessions)
10228}
10229
10230fn next_untried_in_roster<'a>(
10251 roster: &'a [AgentSpec],
10252 start: usize,
10253 tried: &BTreeSet<String>,
10254) -> Option<&'a AgentSpec> {
10255 roster
10256 .get(start + 1..)?
10257 .iter()
10258 .find(|s| !tried.contains(&s.id))
10259}
10260
10261fn pick_successor<'a>(
10282 roster: &'a [AgentSpec],
10283 start: usize,
10284 tried: &BTreeSet<String>,
10285 carried: Option<&BTreeSet<String>>,
10286 others: &BTreeSet<String>,
10287) -> Option<&'a AgentSpec> {
10288 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10289 roster
10290 .get(start + 1..)
10291 .and_then(|tail| {
10292 tail.iter()
10293 .filter(free)
10294 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10295 })
10296 .or_else(|| {
10297 carried
10298 .is_some()
10299 .then(|| roster.iter().find(free))
10300 .flatten()
10301 })
10302 .or_else(|| match carried {
10303 Some(c) => next_for_seat(roster, start, tried, c),
10304 None => next_untried_in_roster(roster, start, tried),
10305 })
10306}
10307
10308fn next_for_seat<'a>(
10318 roster: &'a [AgentSpec],
10319 start: usize,
10320 round_tried: &BTreeSet<String>,
10321 carried_failed: &BTreeSet<String>,
10322) -> Option<&'a AgentSpec> {
10323 roster
10324 .get(start + 1..)?
10325 .iter()
10326 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10327 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10328}
10329
10330fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10336 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10337 return spec;
10338 };
10339 let ok = |id: &str| !h.failed.contains(id);
10340 if let Some(last) = h.last_ok.as_deref()
10341 && ok(last)
10342 && let Some(s) = roster.iter().find(|s| s.id == last)
10343 {
10344 return s.clone();
10345 }
10346 if ok(&spec.id) {
10347 return spec;
10348 }
10349 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10350 roster
10351 .get(from + 1..)
10352 .into_iter()
10353 .flatten()
10354 .chain(roster.iter())
10355 .find(|s| ok(&s.id))
10356 .cloned()
10357 .unwrap_or(spec)
10358}
10359
10360pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10364 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10365}
10366
10367const TIMED_OUT: &str = "timed out";
10370
10371impl FailClass {
10372 fn of(out: &AgentOutcome) -> Option<Self> {
10374 match out {
10375 AgentOutcome::Ok(_) => None,
10376 AgentOutcome::Quota(_) => Some(Self::Quota),
10377 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10378 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10379 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10380 AgentOutcome::ResourceBlocked(_) => None,
10383 }
10384 }
10385
10386 fn stem_word(&self) -> &'static str {
10388 match self {
10389 Self::Quota => "quota",
10390 _ => "handover",
10391 }
10392 }
10393}
10394
10395fn failure_signature(msg: &str) -> String {
10399 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10400 if let Some(code) = line
10403 .strip_prefix("process exited with no usable output (exit code ")
10404 .and_then(|s| s.strip_suffix(')'))
10405 .and_then(|s| s.parse::<i32>().ok())
10406 {
10407 line = format!("exited with some({code}) and no usable output");
10408 } else if matches!(
10409 line.as_str(),
10410 "process exited with no usable output (no exit code reported)"
10411 | "process exited with no usable output (exit code unavailable)"
10412 ) {
10413 line = "exited with none and no usable output".to_owned();
10414 }
10415 let mut out = Vec::new();
10416 for word in line.split_whitespace() {
10417 if word.contains('/') || word.contains('\\') {
10418 out.push("<path>".to_owned());
10419 continue;
10420 }
10421 let mut w = String::new();
10422 let mut in_digits = false;
10423 for c in word.chars() {
10424 if c.is_ascii_digit() {
10425 if !in_digits {
10426 w.push('#');
10427 }
10428 in_digits = true;
10429 } else {
10430 in_digits = false;
10431 w.push(c);
10432 }
10433 }
10434 out.push(w);
10435 }
10436 out.join(" ").chars().take(120).collect()
10437}
10438
10439fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10446 match cur {
10447 FailClass::Quota | FailClass::Timeout => true,
10448 FailClass::Other(_) => prev != Some(cur),
10449 }
10450}
10451
10452fn fail_reason(out: &AgentOutcome) -> String {
10454 match out {
10455 AgentOutcome::Ok(_) => String::new(),
10456 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10457 AgentOutcome::Dropped(o) => format!(
10458 "the CLI dropped the stream ({})",
10459 o.dropped
10460 .as_ref()
10461 .map(|d| d.why.as_str())
10462 .unwrap_or("it ended without delivering its answer")
10463 ),
10464 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10465 e.lines().next().unwrap_or("").chars().take(160).collect()
10466 }
10467 }
10468}
10469
10470pub(crate) fn record_handover(
10473 state: &mut RunState,
10474 node: &str,
10475 seat: &str,
10476 from: &str,
10477 to: &str,
10478 class: &FailClass,
10479 reason: &str,
10480) {
10481 let message = if *class == FailClass::Quota {
10482 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10483 } else {
10484 format!("{seat}: handed over {from} -> {to} ({reason})")
10485 };
10486 state.event(node, message);
10487 state.handovers.push(Handover {
10488 at: Timestamp::now(),
10489 node: node.to_owned(),
10490 seat: seat.to_owned(),
10491 from: from.to_owned(),
10492 to: to.to_owned(),
10493 reason: reason.to_owned(),
10494 });
10495}
10496
10497fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10511 commands.iter().any(|c| c.exit_code.is_none())
10512}
10513
10514fn verified_noop_claim(
10527 usable: bool,
10528 commands: &[agent::CommandEvidence],
10529 text: &str,
10530) -> Option<String> {
10531 (usable && !has_unconfirmed_command(commands))
10532 .then(|| verdict::verified_noop(text))
10533 .flatten()
10534}
10535
10536fn short(commit: &str) -> String {
10537 commit.chars().take(7).collect()
10538}
10539
10540fn make_executable(path: &Path) -> Result<()> {
10541 #[cfg(unix)]
10542 {
10543 use std::os::unix::fs::PermissionsExt as _;
10544 let mut perms = std::fs::metadata(path)?.permissions();
10545 perms.set_mode(0o755);
10546 std::fs::set_permissions(path, perms)?;
10547 }
10548 #[cfg(not(unix))]
10549 {
10550 let _ = path;
10551 }
10552 Ok(())
10553}
10554
10555struct WaveCtx<'a> {
10562 run: &'a str,
10565 node: &'a str,
10567 prompts: &'a Prompts,
10568 cache: Option<&'a Path>,
10570 round: Option<usize>,
10573 carry_seats: bool,
10577 pauses: Option<(&'a Pause, &'a Pause)>,
10580}
10581
10582fn repository_wait(
10584 state: &RunState,
10585 pauses: Option<(&Pause, &Pause)>,
10586) -> crate::repo_lock::WaitPolicy {
10587 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10588 let inherited = crate::repo_lock::WaitPolicy::current();
10589 crate::repo_lock::WaitPolicy::new(
10590 Duration::from_secs(state.config.graph.timeout_resource_wait),
10591 move || {
10592 inherited.cancelled()
10593 || pauses
10594 .as_ref()
10595 .is_some_and(|(a, b)| a.parked() || b.parked())
10596 },
10597 )
10598}
10599
10600fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10601 if error.is::<crate::repo_lock::ResourceBlocked>() {
10602 state.resource_block = Some(crate::run::ResourceBlock {
10603 cause: crate::run::ResourceBlockCause::Other,
10604 node: "repository".into(),
10605 kind: if error.to_string().contains("disk pressure") {
10606 "disk pressure"
10607 } else if error.to_string().contains("disk admission") {
10608 "disk admission"
10609 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10610 "incremental cache admission"
10611 } else {
10612 "repository coordination"
10613 }
10614 .into(),
10615 detail: error.to_string(),
10616 at: Timestamp::now(),
10617 });
10618 state.event("resource", error.to_string());
10619 }
10620}
10621
10622async fn coordinate_operation(
10623 state: &mut RunState,
10624 cwd: &Path,
10625 wait: &crate::repo_lock::WaitPolicy,
10626) -> Result<Option<crate::repo_lock::SeatLease>> {
10627 let result =
10628 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10629 .await;
10630 if let Err(e) = &result {
10631 record_repository_block(state, e);
10632 }
10633 result
10634}
10635
10636async fn run_one(
10638 job: SeatJob,
10639 sem: Arc<Semaphore>,
10640 ctx: &WaveCtx<'_>,
10641 state: &mut RunState,
10642 attempt: usize,
10643) -> (SeatState, AgentOutcome) {
10644 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10645 .await
10646 .pop()
10647 .expect("one job in, one result out");
10648 (seat, out)
10649}
10650
10651async fn wave(
10657 jobs: Vec<SeatJob>,
10658 sem: Arc<Semaphore>,
10659 ctx: &WaveCtx<'_>,
10660 state: &mut RunState,
10661 attempt: usize,
10662) -> Vec<(usize, SeatState, AgentOutcome)> {
10663 let WaveCtx {
10664 run,
10665 node,
10666 prompts,
10667 cache,
10668 round,
10669 carry_seats: _,
10670 pauses,
10671 } = *ctx;
10672 for job in &jobs {
10673 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10674 }
10675 if let Err(e) = state.save() {
10676 tracing::warn!("could not persist in-progress seats: {e:#}");
10681 }
10682 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10698 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10702 let cache_guard = if let Some(cache_dir) = cache {
10703 if jobs_had_a_writer {
10704 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10705 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10706 match acquire_cache_lease(
10707 state,
10708 cache_dir,
10709 &owner,
10710 resource_limit,
10711 node,
10712 node,
10713 &cancel,
10714 )
10715 .await
10716 {
10717 Ok(Some(g)) => Some(g),
10718 Ok(None) => {
10721 if let Some(block) = &mut state.resource_block {
10722 block.cause = crate::run::ResourceBlockCause::Cancelled {
10723 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10724 };
10725 }
10726 return block_wave(
10727 jobs,
10728 state,
10729 node,
10730 attempt,
10731 "parked while waiting for the shared build cache",
10732 );
10733 }
10734 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10737 Err(e) => {
10738 let why = format!("{e:#}");
10742 state.event(
10743 "resource",
10744 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10745 );
10746 return block_wave(jobs, state, node, attempt, &why);
10747 }
10748 }
10749 } else {
10750 None
10751 }
10752 } else {
10753 None
10754 };
10755 state.seats_spawning(node, attempt);
10758 if let Err(e) = state.save() {
10759 tracing::warn!("could not persist the start of a wave: {e:#}");
10760 }
10761 let slot_limit = resource_limit
10765 + jobs
10766 .iter()
10767 .map(|j| j.timeout)
10768 .max()
10769 .unwrap_or_default()
10770 .saturating_mul(jobs.len() as u32);
10771 let mut set = tokio::task::JoinSet::new();
10772 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10775 let overlay = prompts.overlay(node);
10776 let repository_wait = repository_wait(state, pauses);
10777 for (i, mut job) in jobs.into_iter().enumerate() {
10778 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10779 if cache.is_some() {
10780 job.prompt.push('\n');
10781 job.prompt
10782 .push_str(&prompt::build_cache_note(node, job.allow_write));
10783 }
10784 let sem = Arc::clone(&sem);
10785 let run = run.to_owned();
10786 let node = node.to_owned();
10787 let attachments = if node == "implement" {
10790 state.attachments.clone()
10791 } else {
10792 Vec::new()
10793 };
10794 let cache = cache
10805 .filter(|_| job.allow_write && cache_guard.is_some())
10806 .map(Path::to_path_buf);
10807 let spawned_tx = spawned_tx.clone();
10808 let repository_wait = repository_wait.clone();
10809 let coordination = state.config.coordination.clone();
10810 let fetched_base = state.fetched_base.clone();
10811 set.spawn(async move {
10812 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10816 Ok(p) => p,
10817 Err(_) => {
10818 return (
10819 i,
10820 job.seat,
10821 AgentOutcome::ResourceBlocked(format!(
10822 "no parallel slot became free within {}s",
10823 slot_limit.as_secs()
10824 )),
10825 false,
10826 );
10827 }
10828 };
10829 spawned_tx.send(job.seat.key.clone()).ok();
10830 let mut seat = job.seat;
10831 let out = repository_wait
10832 .scope(agent::invoke(
10833 &job.spec,
10834 &mut seat,
10835 &Invocation {
10836 coordination: coordination.as_ref(),
10837 fetched_base: fetched_base.as_ref(),
10838 cwd: &job.cwd,
10839 prompt: &job.prompt,
10840 timeout: job.timeout,
10841 allow_write: job.allow_write,
10842 sessions: job.sessions,
10843 artifacts: &job.artifacts,
10844 stem: &job.stem,
10845 run: &run,
10846 node: &node,
10847 cache_dir: cache.as_deref(),
10848 attachments: &attachments,
10849 writable: &[],
10850 },
10851 ))
10852 .await;
10853 let interrupted = out
10854 .as_ref()
10855 .err()
10856 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10857 let out = match out {
10858 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10859 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10860 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10868 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10869 Ok(o) => AgentOutcome::Failed(format!(
10870 "process exited with no usable output ({})",
10871 crate::notices::exit_status(o.exit_code)
10872 )),
10873 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10874 AgentOutcome::ResourceBlocked(e.to_string())
10875 }
10876 Err(e) => AgentOutcome::Failed(e.to_string()),
10877 };
10878 (i, seat, out, interrupted)
10879 });
10880 }
10881 drop(spawned_tx);
10882 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10883 loop {
10884 let joined = tokio::select! {
10885 Some(key) = spawned_rx.recv() => {
10886 state.seat_spawned(&key);
10888 if let Err(e) = state.save() {
10889 tracing::warn!("could not persist a seat's start: {e:#}");
10890 }
10891 continue;
10892 }
10893 joined = set.join_next() => joined,
10894 };
10895 let Some(joined) = joined else {
10896 break;
10897 };
10898 let (i, seat, out, interrupted) = match joined {
10899 Ok(v) => v,
10900 Err(e) => {
10904 tracing::error!("agent task panicked: {e}");
10905 continue;
10906 }
10907 };
10908 state.seat_finished(&seat.key);
10909 if interrupted
10910 && node == "implement"
10911 && let Some(label) = state
10912 .candidates
10913 .iter()
10914 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10915 .map(|candidate| candidate.label)
10916 {
10917 state.candidate_preservation.insert(
10918 label,
10919 "interrupted implementation awaiting persistence".into(),
10920 );
10921 state.interrupted_implementations.insert(label);
10922 }
10923 if let AgentOutcome::ResourceBlocked(why) = &out
10924 && state.resource_block.is_none()
10925 {
10926 state.resource_block = Some(crate::run::ResourceBlock {
10927 cause: crate::run::ResourceBlockCause::Other,
10928 node: node.to_owned(),
10929 kind: if why.contains("disk pressure") {
10930 "disk pressure"
10931 } else if why.contains("disk admission") {
10932 "disk admission"
10933 } else if why.contains(crate::incremental::BLOCKED) {
10934 "incremental cache admission"
10935 } else if why.contains("repository coordination") {
10936 "repository coordination"
10937 } else {
10938 "parallel slot"
10939 }
10940 .to_owned(),
10941 detail: why.clone(),
10942 at: Timestamp::now(),
10943 });
10944 let activity = if interrupted {
10945 "agent execution interrupted"
10946 } else {
10947 "no agent was started"
10948 };
10949 state.event("resource", format!("{node}: {activity} - {why}"));
10950 }
10951 record_jobs(state, node, round, &seat.key, &out);
10952 if let Err(e) = state.save() {
10953 tracing::warn!("could not persist a seat's completion: {e:#}");
10954 }
10955 if collected.len() <= i {
10956 collected.resize_with(i + 1, || None);
10957 }
10958 collected[i] = Some((i, seat, out));
10959 }
10960 if state
10966 .active
10967 .values()
10968 .any(|a| a.node == node && a.attempt == attempt)
10969 {
10970 state
10971 .active
10972 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10973 if let Err(e) = state.save() {
10974 tracing::warn!("could not persist the end of a wave: {e:#}");
10975 }
10976 }
10977 if let Some(cache_dir) = cache
10984 && jobs_had_a_writer
10985 {
10986 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10987 }
10988 if let Some(guard) = cache_guard {
10989 guard.release();
10990 }
10991 collected.into_iter().flatten().collect()
10992}
10993
10994fn record_jobs(
11005 state: &mut RunState,
11006 node: &str,
11007 round: Option<usize>,
11008 seat: &str,
11009 out: &AgentOutcome,
11010) {
11011 let commands: &[agent::CommandEvidence] = match out {
11012 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11013 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11014 };
11015 let checked_at = Timestamp::now();
11016 for c in commands {
11017 state.jobs.push(JobRecord {
11018 node: node.to_owned(),
11019 round,
11020 seat: seat.to_owned(),
11021 id: c.id.clone(),
11022 description: c.description.clone(),
11023 checked_at,
11024 status: match c.exit_code {
11025 Some(0) => JobStatus::Completed,
11026 Some(_) => JobStatus::Failed,
11027 None => JobStatus::Unknown,
11028 },
11029 exit_code: c.exit_code,
11030 result_summary: c.result_summary.clone(),
11031 source: c.source.clone(),
11032 });
11033 }
11034}
11035
11036fn round_is_clean(
11057 blocking: usize,
11058 e2e_ok: bool,
11059 answered: usize,
11060 expected: usize,
11061 quota_missing: usize,
11062 policy: IncompleteReviewPolicy,
11063) -> bool {
11064 if blocking != 0 || !e2e_ok {
11065 return false;
11066 }
11067 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11068 return true;
11069 }
11070 answered > 0 && expected - answered <= quota_missing
11071}
11072
11073pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11097 if max_rounds == 0
11098 || reviews.last().is_some_and(|r| {
11099 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11100 })
11101 {
11102 return Some(RunStatus::Gating);
11103 }
11104 let last = reviews.last()?;
11105 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11106 if reviews.len() < max_rounds && !stagnant {
11107 return None;
11108 }
11109 if last.incomplete() && last.blocking == 0 {
11110 return Some(RunStatus::Blocked);
11111 }
11112 if last.e2e_status() == E2eStatus::ResourceBlocked
11113 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11114 {
11115 return None;
11116 }
11117 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11118 RunStatus::Gating
11119 } else {
11120 RunStatus::Blocked
11121 })
11122}
11123
11124pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11139 if nudged {
11140 (full / 4).max(Duration::from_secs(120)).min(full)
11141 } else {
11142 full
11143 }
11144}
11145
11146#[allow(clippy::too_many_arguments)]
11175async fn ask_json_wave<T>(
11176 jobs: Vec<SeatJob>,
11177 sem: Arc<Semaphore>,
11178 retries: usize,
11179 roster: &[AgentSpec],
11180 ctx: &WaveCtx<'_>,
11181 losses: &mut Vec<QuotaLoss>,
11182 state: &mut RunState,
11183 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11184) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11185where
11186 T: serde::de::DeserializeOwned + Send + 'static,
11187{
11188 ask_wave_with(
11189 jobs,
11190 sem,
11191 retries,
11192 roster,
11193 ctx,
11194 losses,
11195 state,
11196 &|text: &str| {
11197 let v = verdict::extract_json::<T>(text)?;
11198 validate(&v)?;
11199 Ok(v)
11200 },
11201 )
11202 .await
11203}
11204
11205#[allow(clippy::too_many_arguments)]
11214async fn ask_wave_with<T>(
11215 jobs: Vec<SeatJob>,
11216 sem: Arc<Semaphore>,
11217 retries: usize,
11218 roster: &[AgentSpec],
11219 ctx: &WaveCtx<'_>,
11220 losses: &mut Vec<QuotaLoss>,
11221 state: &mut RunState,
11222 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11223) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11224where
11225 T: Send + 'static,
11226{
11227 let n = jobs.len();
11228 let originals: Vec<SeatJob> = jobs;
11229 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11230 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11231 let mut nudges: Vec<usize> = vec![0; n];
11238 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11242 let mut tried: Vec<BTreeSet<String>> = specs
11243 .iter()
11244 .map(|s| BTreeSet::from([s.id.clone()]))
11245 .collect();
11246 let starts: Vec<usize> = specs
11247 .iter()
11248 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11249 .collect();
11250 let carry = ctx.carry_seats && !roster.is_empty();
11251 let carried: Vec<BTreeSet<String>> = originals
11254 .iter()
11255 .map(|j| {
11256 state
11257 .seat_history
11258 .get(&j.seat.key)
11259 .filter(|_| carry)
11260 .map(|h| h.failed.clone())
11261 .unwrap_or_default()
11262 })
11263 .collect();
11264 let mut prev: Vec<Option<FailClass>> = originals
11265 .iter()
11266 .map(|j| {
11267 state
11268 .seat_history
11269 .get(&j.seat.key)
11270 .filter(|_| carry)
11271 .and_then(|h| h.last_fail.clone())
11272 })
11273 .collect();
11274 let next_agent =
11275 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11276 let others: BTreeSet<String> = specs
11277 .iter()
11278 .enumerate()
11279 .filter(|(j, _)| *j != i)
11280 .map(|(_, s)| s.id.clone())
11281 .collect();
11282 pick_successor(
11283 roster,
11284 starts[i],
11285 tried,
11286 carry.then(|| &carried[i]),
11287 &others,
11288 )
11289 .cloned()
11290 };
11291 let mut fresh: Vec<Option<String>> = vec![None; n];
11292 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11293 let mut pending: Vec<usize> = (0..n).collect();
11294
11295 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11300 for round in 0..max_rounds {
11301 if pending.is_empty() {
11302 break;
11303 }
11304 let mut batch = Vec::with_capacity(pending.len());
11305 let mut renudged: Vec<&str> = Vec::new();
11306 for &i in &pending {
11307 let src = &originals[i];
11308 let full: &str = match &src.handover {
11313 Some(h) if specs[i].id != src.spec.id => h,
11314 _ => &src.prompt,
11315 };
11316 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11319 (
11320 full.to_owned(),
11321 src.timeout,
11322 format!("{}-{word}-{}", src.stem, specs[i].id),
11323 )
11324 } else if nudges[i] == 0 {
11325 (full.to_owned(), src.timeout, src.stem.clone())
11326 } else {
11327 renudged.push(src.seat.key.as_str());
11328 let why = done[i]
11329 .as_ref()
11330 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11331 .unwrap_or_else(|| "no parsable answer".to_owned());
11332 let nudge = prompt::nudge(&why);
11333 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11334 let prompt = if nudged {
11335 nudge
11336 } else {
11337 format!("{full}\n\n---\n\n{nudge}")
11338 };
11339 (
11340 prompt,
11341 retry_budget(src.timeout, nudged),
11342 format!("{}-retry{}", src.stem, nudges[i]),
11343 )
11344 };
11345 batch.push(SeatJob {
11346 spec: specs[i].clone(),
11347 seat: seats[i].clone(),
11348 cwd: src.cwd.clone(),
11349 prompt,
11350 timeout,
11351 allow_write: src.allow_write,
11352 sessions: src.sessions,
11353 artifacts: src.artifacts.clone(),
11354 stem,
11355 handover: None,
11356 });
11357 }
11358
11359 if !renudged.is_empty() {
11360 state.event(
11361 ctx.node,
11362 format!("retry {round}: re-asking {}", renudged.join(", ")),
11363 );
11364 }
11365 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11366 let mut still = Vec::new();
11367 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11368 seats[i] = seat;
11369 if let AgentOutcome::ResourceBlocked(why) = &out {
11372 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11373 continue;
11374 }
11375 let class = FailClass::of(&out);
11376 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11379 && nudges[i] < retries
11380 && !roster.is_empty();
11381 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11382 let next = should_hand_over(prev[i].as_ref(), &cur)
11383 .then(|| next_agent(i, &tried[i], &specs))
11384 .flatten();
11385 if carry {
11386 let h = state
11387 .seat_history
11388 .entry(originals[i].seat.key.clone())
11389 .or_default();
11390 h.failed.insert(specs[i].id.clone());
11391 h.last_fail = Some(cur.clone());
11392 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11393 h.last_ok = None;
11394 }
11395 if let Err(e) = state.save() {
11398 tracing::warn!("could not persist a seat's failure history: {e:#}");
11399 }
11400 }
11401 if let Some(next) = next {
11402 record_handover(
11403 state,
11404 ctx.node,
11405 &originals[i].seat.key,
11406 &specs[i].id,
11407 &next.id,
11408 &cur,
11409 &fail_reason(&out),
11410 );
11411 tried[i].insert(next.id.clone());
11412 prev[i] = Some(cur.clone());
11413 seats[i] =
11414 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11415 specs[i] = next;
11416 fresh[i] = Some(cur.stem_word().to_owned());
11417 nudges[i] = 0;
11418 done[i] = Some(Err(anyhow::anyhow!(
11419 "handed over after: {}",
11420 fail_reason(&out)
11421 )));
11422 still.push(i);
11423 continue;
11424 }
11425 }
11426 if carry
11427 && !nudge_first
11428 && let Some(cur) = class.clone()
11429 {
11430 let h = state
11432 .seat_history
11433 .entry(originals[i].seat.key.clone())
11434 .or_default();
11435 h.failed.insert(specs[i].id.clone());
11436 h.last_fail = Some(cur);
11437 }
11438 let parsed = match out {
11439 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11440 AgentOutcome::Quota(o) => {
11441 last_quota[i] = o.quota.clone();
11442 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11443 }
11444 AgentOutcome::Dropped(o) => {
11448 let why = o
11449 .dropped
11450 .as_ref()
11451 .map(|d| d.why.as_str())
11452 .unwrap_or("the CLI ended the stream without delivering its answer");
11453 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11454 }
11455 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11456 Err(anyhow::anyhow!(e))
11457 }
11458 };
11459 let quota = class == Some(FailClass::Quota);
11460 let failed = parsed.is_err();
11461 if !failed && matches!(ctx.node, "judge" | "vote") {
11462 state
11465 .quota
11466 .retain(|loss| loss.seat != originals[i].seat.key);
11467 }
11468 done[i] = Some(parsed);
11469 if carry && !failed {
11470 let h = state
11471 .seat_history
11472 .entry(originals[i].seat.key.clone())
11473 .or_default();
11474 h.failed.remove(&specs[i].id);
11475 h.last_ok = Some(specs[i].id.clone());
11476 h.last_fail = None;
11477 }
11478 let agent_failure = class.is_some()
11486 && !nudge_first
11487 && !roster.is_empty()
11488 && next_agent(i, &tried[i], &specs).is_some();
11489 if failed && !quota && !agent_failure && nudges[i] < retries {
11490 nudges[i] += 1;
11491 still.push(i);
11492 }
11493 }
11494 pending = still;
11495 }
11496
11497 for (i, q) in last_quota.into_iter().enumerate() {
11500 if let Some(quota) = q {
11501 losses.push(QuotaLoss::from_report(
11502 originals[i].seat.key.clone(),
11503 ctx.node,
11504 Some("a),
11505 ));
11506 }
11507 }
11508
11509 if state.status != RunStatus::Stalled
11512 && ctx.node != "deliberate"
11513 && n > 0
11514 && losses.len() == n
11515 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11516 {
11517 block_machine_outage(state, ctx.node, losses);
11518 state.quota.append(losses);
11519 }
11520
11521 seats
11522 .into_iter()
11523 .zip(done)
11524 .zip(nudges)
11525 .map(|((seat, res), attempts)| {
11526 (
11527 seat,
11528 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11529 attempts,
11530 )
11531 })
11532 .collect()
11533}
11534
11535pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11538 let mut auth: Vec<_> = losses
11539 .iter()
11540 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11541 .map(|loss| loss.agent.clone())
11542 .collect();
11543 auth.sort();
11544 auth.dedup();
11545 let (kind, detail) = if auth.is_empty() {
11546 (
11547 "agent usage limit",
11548 "All seats reached their agent usage limits; resume after usage is available"
11549 .to_owned(),
11550 )
11551 } else {
11552 (
11553 "agent authentication",
11554 format!(
11555 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11556 auth.join(", ")
11557 ),
11558 )
11559 };
11560 state.resource_block = Some(crate::run::ResourceBlock {
11561 node: node.to_owned(),
11562 kind: kind.to_owned(),
11563 cause: crate::run::ResourceBlockCause::Other,
11564 detail: detail.clone(),
11565 at: Timestamp::now(),
11566 });
11567 state.event(node, detail);
11568}
11569
11570fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11572 results
11573 .iter()
11574 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11575}
11576
11577fn block_wave(
11581 jobs: Vec<SeatJob>,
11582 state: &mut RunState,
11583 node: &str,
11584 attempt: usize,
11585 why: &str,
11586) -> Vec<(usize, SeatState, AgentOutcome)> {
11587 state.resource_wait = None;
11588 state
11589 .active
11590 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11591 if let Err(e) = state.save() {
11592 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11593 }
11594 jobs.into_iter()
11595 .enumerate()
11596 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11597 .collect()
11598}
11599
11600async fn acquire_cache_lease(
11613 state: &mut RunState,
11614 cache_dir: &Path,
11615 owner: &crate::cache::Owner,
11616 budget: Duration,
11617 node: &str,
11618 context: &str,
11619 cancel: &(dyn Fn() -> bool + Sync),
11620) -> Result<Option<crate::cache::Guard>> {
11621 let home = crate::run::home();
11622 let started = Instant::now();
11623 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11624 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11625 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11626 Err(e) => {
11627 state.event(
11628 "verify",
11629 format!("{context}: could not check the shared build cache: {e:#}"),
11630 );
11631 if let Err(e2) = state.save() {
11632 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11633 }
11634 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11635 }
11636 };
11637 state.event(
11638 "verify",
11639 format!(
11640 "{context}: waiting for the shared build cache at {} ({})",
11641 cache_dir.display(),
11642 busy.describe()
11643 ),
11644 );
11645 state.resource_wait = Some(crate::run::ResourceWait {
11648 node: context.to_owned(),
11649 kind: "shared build cache".to_owned(),
11650 holder: busy.describe(),
11651 since: Timestamp::now(),
11652 limit_secs: budget.as_secs(),
11653 });
11654 if let Err(e) = state.save() {
11655 tracing::warn!("could not persist a cache wait: {e:#}");
11656 }
11657 let remaining = budget.saturating_sub(started.elapsed());
11658 let waited = crate::cache::wait_for_unless(
11659 &home,
11660 cache_dir,
11661 owner,
11662 remaining,
11663 Duration::from_secs(5),
11664 cancel,
11665 )
11666 .await;
11667 state.resource_wait = None;
11668 let cause = match &waited {
11669 Ok(Some(_)) => None,
11670 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11671 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11672 Some(crate::run::ResourceBlockCause::Timeout {
11673 waited_secs: started.elapsed().as_secs(),
11674 limit_secs: budget.as_secs(),
11675 })
11676 }
11677 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11678 };
11679 if let Some(cause) = cause {
11680 let detail = match &waited {
11681 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11682 Err(error) => format!("{error:#}"),
11683 };
11684 state.resource_block = Some(crate::run::ResourceBlock {
11685 cause,
11686 node: node.to_owned(),
11687 kind: "shared build cache".into(),
11688 detail,
11689 at: Timestamp::now(),
11690 });
11691 state.event(
11692 "resource",
11693 state.resource_block.as_ref().unwrap().description(),
11694 );
11695 }
11696 if let Err(error) = state.save() {
11697 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11698 }
11699 waited
11700}
11701
11702#[derive(Debug)]
11707struct CacheCheckError(String);
11708
11709impl std::fmt::Display for CacheCheckError {
11710 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11711 f.write_str(&self.0)
11712 }
11713}
11714
11715impl std::error::Error for CacheCheckError {}
11716
11717fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11719 state
11720 .resource_block
11721 .get_or_insert_with(|| crate::run::ResourceBlock {
11722 cause: crate::run::ResourceBlockCause::Other,
11723 node: node.to_owned(),
11724 kind: "shared build cache".into(),
11725 detail: format!("{error:#}"),
11726 at: Timestamp::now(),
11727 });
11728 if let Err(error) = state.save() {
11729 tracing::warn!("could not persist cache resource block: {error:#}");
11730 }
11731}
11732
11733#[allow(clippy::too_many_arguments)]
11754async fn with_cache_lease<'s, F, Fut>(
11755 state: &'s mut RunState,
11756 cache_dir: Option<&Path>,
11757 node: &str,
11758 seat: &str,
11759 worktree: &Path,
11760 head: &str,
11761 budget: Duration,
11762 context: &str,
11763 wait: &crate::repo_lock::WaitPolicy,
11764 body: F,
11765) -> Result<(Vec<CommandOutcome>, bool)>
11766where
11767 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11768 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11769{
11770 let started = Instant::now();
11773 let home = crate::run::home();
11774 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11775 let guard = if let Some(cache_dir) = cache_dir {
11776 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11777 match acquire_cache_lease(
11778 state,
11779 cache_dir,
11780 &owner,
11781 resource_limit,
11782 node,
11783 context,
11784 &|| wait.cancelled(),
11785 )
11786 .await
11787 .and_then(|g| {
11788 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11789 }) {
11790 Ok(g) => Some(g),
11791 Err(e) => {
11792 record_cache_error(state, node, &e);
11793 return Ok((
11794 vec![CommandOutcome {
11795 command: "(waiting for the shared build cache)".into(),
11796 code: None,
11797 output_tail: e.to_string(),
11798 duration_ms: started.elapsed().as_millis() as u64,
11799 resource_blocked: true,
11800 }],
11801 false,
11802 ));
11803 }
11804 }
11805 } else {
11806 None
11807 };
11808 let _coordination = coordinate_operation(state, worktree, wait).await?;
11809 if let Some(cache_dir) = cache_dir {
11810 let identity = crate::cache::Identity::new(worktree, head);
11811 if let Err(e) = crate::cache::ensure_fresh(
11812 &home,
11813 cache_dir,
11814 &identity,
11815 &state.config.verify.cargo_manifests(),
11816 ) {
11817 record_cache_error(state, node, &e);
11818 state.event(
11819 "verify",
11820 format!(
11821 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11822 worktree.display(),
11823 short(head)
11824 ),
11825 );
11826 if let Some(guard) = guard {
11827 guard.release();
11828 }
11829 return Ok((
11830 vec![CommandOutcome {
11831 command: "(confirming the shared build cache is fresh)".into(),
11832 code: None,
11833 output_tail: e.to_string(),
11834 duration_ms: started.elapsed().as_millis() as u64,
11835 resource_blocked: true,
11836 }],
11837 false,
11838 ));
11839 }
11840 }
11841 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11842 if !timed_out_pids.is_empty() {
11843 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11844 }
11845 drop(_coordination);
11848 if let Some(guard) = guard {
11849 guard.release();
11850 }
11851 Ok((outcomes, retried))
11852}
11853
11854async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11866 wait_for_pids_with(
11867 pids,
11868 crate::proc::pid_alive,
11869 LEASE_RELEASE_POLL,
11870 LEASE_RELEASE_MAX_WAIT,
11871 )
11872 .await;
11873}
11874
11875async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11881 pids: &[u32],
11882 alive: F,
11883 poll: Duration,
11884 max_wait: Duration,
11885) {
11886 let deadline = Instant::now() + max_wait;
11887 loop {
11888 if pids.iter().all(|&pid| !alive(pid)) {
11889 return;
11890 }
11891 if Instant::now() >= deadline {
11892 return;
11893 }
11894 tokio::time::sleep(poll).await;
11895 }
11896}
11897
11898fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11905 outcomes.iter().any(|o| o.resource_blocked)
11906}
11907
11908pub(crate) enum GateFix {
11910 Retry,
11912 Stop,
11915 Defer,
11918}
11919
11920pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11928 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11929 red.peek().is_some()
11930 && red.all(|o| {
11931 !o.resource_blocked
11932 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11933 && !o.output_tail.trim().is_empty()
11934 })
11935}
11936
11937fn e2e_outcome_label(o: &CommandOutcome) -> String {
11941 if o.ok() {
11942 return "pass".to_owned();
11943 }
11944 let reason = if o.build_failed() {
11945 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11946 } else {
11947 format!("FAIL ({:?})", o.code)
11948 };
11949 format!(
11950 "{reason}\n{}",
11951 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11952 )
11953}
11954
11955async fn run_e2e_with_retry(
11963 state: &mut RunState,
11964 shell: &[String],
11965 commands: &[String],
11966 worktree: &Path,
11967 timeout: Duration,
11968 context: &str,
11969) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11970 let (mut e2e, mut timed_out_pids) = run_commands(
11971 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11972 )
11973 .await;
11974 for o in &e2e {
11975 state.event(
11976 "verify",
11977 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11978 );
11979 }
11980 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11984 if verify_retried {
11985 state.event(
11986 "verify",
11987 format!(
11988 "{context}: verify could not build/link, not a test result — retrying once \
11989 before concluding"
11990 ),
11991 );
11992 let retried = run_commands(
11993 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11994 )
11995 .await;
11996 e2e = retried.0;
11997 timed_out_pids.extend(retried.1);
12000 for o in &e2e {
12001 state.event(
12002 "verify",
12003 format!(
12004 "{context}: retry `{}` -> {}",
12005 o.command,
12006 e2e_outcome_label(o)
12007 ),
12008 );
12009 }
12010 }
12011 (e2e, verify_retried, timed_out_pids)
12012}
12013
12014#[allow(clippy::too_many_arguments)]
12030async fn run_commands(
12031 state: &mut RunState,
12032 node: &str,
12033 task: &str,
12034 attempt: usize,
12035 shell: &[String],
12036 commands: &[String],
12037 cwd: &Path,
12038 timeout: Duration,
12039) -> (Vec<CommandOutcome>, Vec<u32>) {
12040 if commands.is_empty() {
12041 return (Vec::new(), Vec::new());
12046 }
12047 let mut out = Vec::new();
12048 let mut timed_out_pids = Vec::new();
12049 let total = commands.len();
12050 for (idx, command) in commands.iter().enumerate() {
12051 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12052 if let Err(e) = state.save() {
12053 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12054 }
12055 let started = Instant::now();
12056 let mut cmd = tokio::process::Command::new(&shell[0]);
12057 cmd.quiet();
12058 for key in crate::worker::INVOCATION_ENV {
12062 cmd.env_remove(key);
12063 }
12064 cmd.args(&shell[1..])
12065 .arg(command)
12066 .current_dir(cwd)
12067 .stdin(std::process::Stdio::null())
12068 .stdout(std::process::Stdio::piped())
12069 .stderr(std::process::Stdio::piped())
12070 .kill_on_drop(true);
12071 let mut outputs = Vec::new();
12072 let protected = (|| -> Result<()> {
12073 crate::incremental::configure(&mut cmd, cwd)?;
12074 let mut paths = vec![cwd.to_owned()];
12075 if let Some(cache) = state.config.cache_dir() {
12076 paths.push(cache);
12077 }
12078 outputs.extend(crate::retention::OutputLease::admit(
12079 &crate::run::home(),
12080 &paths,
12081 state.config.disk.min_free_bytes,
12082 )?);
12083 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12084 outputs.push(guard);
12085 }
12086 if let Some(cache) = state.config.cache_dir() {
12087 outputs.push(crate::retention::OutputLease::cache(
12088 &crate::run::home(),
12089 &cache,
12090 false,
12091 )?);
12092 }
12093 Ok(())
12094 })();
12095 if let Err(e) = protected {
12096 let detail = format!("execution admission unavailable: {e:#}");
12097 state.resource_block = Some(crate::run::ResourceBlock {
12098 cause: crate::run::ResourceBlockCause::Other,
12099 node: node.to_owned(),
12100 kind: if detail.contains("disk admission") {
12101 "disk admission"
12102 } else {
12103 "output ownership"
12104 }
12105 .to_owned(),
12106 detail: detail.clone(),
12107 at: Timestamp::now(),
12108 });
12109 out.push(CommandOutcome {
12110 command: command.clone(),
12111 code: None,
12112 output_tail: detail,
12113 duration_ms: 0,
12114 resource_blocked: true,
12115 });
12116 break;
12117 }
12118 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12119 let (code, body) = match spawned {
12120 Ok(mut running) => {
12121 let pid = running.child().id();
12122 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12123 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12124 let mut paths = vec![cwd.to_owned()];
12125 if let Some(cache) = state.config.cache_dir() {
12126 paths.push(cache);
12127 }
12128 let mut pressure = None;
12129 let status = tokio::select! {
12130 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12131 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12132 pressure = Some(reason); None
12133 }
12134 };
12135 let (code, failure) = match status {
12136 Some(Ok(Ok(status))) => (
12137 status.code(),
12138 status
12139 .code()
12140 .is_none()
12141 .then(|| format!("command terminated: {status}")),
12142 ),
12143 Some(Ok(Err(error))) => {
12144 (None, Some(format!("failed waiting for command: {error}")))
12145 }
12146 Some(Err(_)) => {
12147 if !running.coordinated()
12151 && let Some(pid) = pid
12152 {
12153 timed_out_pids.push(pid);
12154 }
12155 (
12156 None,
12157 Some(format!("timed out after {}s", timeout.as_secs())),
12158 )
12159 }
12160 None => (None, None),
12161 };
12162 running.finish().await;
12163 if let Some(reason) = pressure {
12164 state.resource_block = Some(crate::run::ResourceBlock {
12165 cause: crate::run::ResourceBlockCause::Other,
12166 node: node.to_owned(),
12167 kind: "disk pressure".to_owned(),
12168 detail: reason.clone(),
12169 at: Timestamp::now(),
12170 });
12171 out.push(CommandOutcome {
12172 command: command.clone(),
12173 code: None,
12174 output_tail: reason,
12175 duration_ms: started.elapsed().as_millis() as u64,
12176 resource_blocked: true,
12177 });
12178 break;
12179 }
12180 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12181 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12182 if let Some(failure) = failure {
12183 body.push('\n');
12184 body.push_str(&failure);
12185 }
12186 (code, body)
12187 }
12188 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12189 };
12190 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12191 if !body.is_empty() {
12192 match crate::run::write_verification_artifact(state, task, &body) {
12196 Ok(name) if summary != body => {
12197 summary = format!("[full output: artifacts/{name}]\n{summary}");
12198 }
12199 Ok(_) => {}
12200 Err(e) => {
12201 let warning = format!("could not save the full {task} output: {e:#}");
12202 tracing::warn!("{warning}");
12203 state.event(node, warning);
12204 if summary != body {
12205 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12206 }
12207 }
12208 }
12209 }
12210 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12211 out.push(CommandOutcome {
12212 command: command.clone(),
12213 code,
12214 output_tail: summary,
12215 duration_ms: started.elapsed().as_millis() as u64,
12216 resource_blocked,
12217 });
12218 if resource_blocked {
12219 state.resource_block = Some(crate::run::ResourceBlock {
12220 cause: crate::run::ResourceBlockCause::Other,
12221 node: node.to_owned(),
12222 kind: "incremental cache admission".into(),
12223 detail: body,
12224 at: Timestamp::now(),
12225 });
12226 break;
12227 }
12228 }
12229 state.task_finished(task);
12230 if let Err(e) = state.save() {
12231 tracing::warn!("could not persist the end of {task}: {e:#}");
12232 }
12233 (out, timed_out_pids)
12234}
12235
12236fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12248 let repo = repo.display();
12249 match style {
12250 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12251 MergeStyle::Squash => {
12252 let subject = message
12255 .lines()
12256 .next()
12257 .unwrap_or(branch)
12258 .replace(['\\', '"', '$', '`'], "");
12259 format!(
12260 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12261 )
12262 }
12263 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12264 }
12265}
12266
12267const PR_TITLE_MAX: usize = 240;
12278
12279fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12290 let line = queue::first_line(text)?;
12291 let chars: Vec<char> = line.chars().collect();
12292 let (kept, mark) = if chars.len() <= max {
12293 (line.to_owned(), "")
12294 } else if let Some(end) = sentence_end(&chars, max) {
12295 (chars[..end].iter().collect::<String>(), "")
12296 } else {
12297 let room = max.saturating_sub(3);
12298 let cut = if chars[room].is_whitespace() {
12301 room
12302 } else {
12303 chars[..room]
12304 .iter()
12305 .rposition(|c| c.is_whitespace())
12306 .unwrap_or(room)
12307 };
12308 let head: String = chars[..cut].iter().collect();
12309 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12310 (head.to_owned(), "...")
12311 };
12312 if kept.is_empty() {
12313 return None;
12314 }
12315 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12316 return None;
12317 }
12318 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12319 return None;
12320 }
12321 Some(format!("{kept}{mark}"))
12322}
12323
12324fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12328 const MIN: usize = 20;
12329 for i in MIN..max.min(chars.len()) {
12330 if !matches!(chars[i], '.' | '!' | '?') {
12331 continue;
12332 }
12333 let Some(&next) = chars.get(i + 1) else {
12334 continue;
12335 };
12336 if !next.is_whitespace() {
12337 continue;
12338 }
12339 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12340 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12341 continue;
12342 }
12343 let word: String = chars[..i]
12344 .iter()
12345 .rev()
12346 .take_while(|c| !c.is_whitespace())
12347 .collect::<Vec<_>>()
12348 .into_iter()
12349 .rev()
12350 .collect();
12351 let word = word.to_ascii_lowercase();
12352 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12353 continue;
12354 }
12355 let end = chars[..i]
12356 .iter()
12357 .rposition(|c| !c.is_whitespace())
12358 .map_or(i, |p| p + 1);
12359 return Some(end);
12360 }
12361 None
12362}
12363
12364struct PrMessage {
12369 title: String,
12370 body: String,
12371}
12372
12373impl PrMessage {
12374 fn commit_message(&self) -> String {
12378 format!("{}\n\n{}", self.title, self.body)
12379 }
12380}
12381
12382fn title_marker(line: &str) -> Option<&str> {
12384 let line = line.trim();
12385 let head = line.get(..6)?;
12386 head.eq_ignore_ascii_case("title:")
12387 .then(|| line[6..].trim())
12388}
12389
12390fn summary_title(summary: &str) -> Option<String> {
12395 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12396 let raw = title_marker(first)?;
12397 if raw.is_empty() {
12398 return None;
12399 }
12400 let title = queue::title_from(raw, PR_TITLE_MAX);
12401 let lower = title.to_ascii_lowercase();
12402 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12403 return None;
12404 }
12405 Some(title)
12406}
12407
12408const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12410
12411pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12413
12414pub(crate) fn is_review_run(state: &RunState) -> bool {
12418 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12419}
12420
12421fn review_title(state: &RunState) -> Option<String> {
12426 english_subject(
12427 state.reviewed_commits.as_ref()?.first()?,
12428 state.config.graph.github_english(),
12429 )
12430}
12431
12432fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12435 let raw = raw.trim();
12436 if raw.is_empty() {
12437 return None;
12438 }
12439 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12440 return None;
12441 }
12442 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12443 return None;
12444 }
12445 let title = queue::title_from(raw, PR_TITLE_MAX);
12446 let lower = title.to_ascii_lowercase();
12447 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12448 return None;
12449 }
12450 Some(title)
12451}
12452
12453#[derive(Debug, Clone, PartialEq, Eq)]
12456struct BranchFacts {
12457 commits: Vec<(String, String)>,
12459 stat: String,
12461}
12462
12463const STAT_FILE_LINES: usize = 25;
12465const COMMITS_MAX_CHARS: usize = 20_000;
12467
12468async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12471 let commits = git::commit_log(repo, base, branch).await.ok()?;
12472 if commits.is_empty() {
12473 return None;
12474 }
12475 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12476 let lines: Vec<&str> = stat.lines().collect();
12477 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12478 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12479 let more = format!(" ... {omitted} more file(s)");
12480 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12481 kept.push(&more);
12482 kept.push(lines[lines.len() - 1]);
12483 kept.join("\n")
12484 } else {
12485 lines.join("\n")
12486 };
12487 Some(BranchFacts { commits, stat })
12488}
12489
12490fn markdown_safe(text: &str) -> String {
12493 text.replace("</details>", "</details>")
12494 .replace("\x60\x60\x60", "~~~")
12495}
12496
12497fn neutral_title(state: &RunState, winner: char) -> String {
12498 format!(
12499 "chore: land candidate {} of run {}",
12500 winner.to_ascii_uppercase(),
12501 state.id
12502 )
12503}
12504
12505pub(crate) async fn repair_landing_gate(
12508 state: &mut RunState,
12509 winner: &Candidate,
12510 outcomes: &[CommandOutcome],
12511) -> Result<GateFix> {
12512 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12513 return Ok(GateFix::Stop);
12514 }
12515 let mut runner = Runner {
12516 roles: state.config.resolve_roles()?,
12517 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12518 state: state.clone(),
12519 pause: Pause::new(),
12520 interrupt: Pause::new(),
12521 };
12522 let result = runner.gate_fix_round(winner, outcomes).await;
12523 *state = runner.state;
12524 result
12525}
12526
12527pub(crate) async fn verify_current_head(
12530 state: &mut RunState,
12531 winner: &Candidate,
12532 wait: &crate::repo_lock::WaitPolicy,
12533) -> Result<bool> {
12534 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12535 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12537 state
12538 .verification_history
12539 .push(crate::run::VerificationEvidence {
12540 superseded_by: Some(head.clone()),
12541 archived_at: Some(Timestamp::now()),
12542 fixes_spent: Some(state.gate_fixes.len()),
12543 gate_head: None,
12544 gate_ran: false,
12545 gate: Vec::new(),
12546 e2e: state.head_e2e.take(),
12547 });
12548 }
12549 if let Some(e) = &state.head_e2e
12550 && e.head == head
12551 {
12552 if e.passed {
12553 return Ok(true);
12554 }
12555 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12556 state.status = RunStatus::Blocked;
12557 state.save()?;
12558 return Ok(false);
12559 }
12560 } else if state.reviews.last().is_some_and(|r| {
12561 r.verified_head.as_deref() == Some(head.as_str())
12562 && (r.e2e_status() == E2eStatus::Passed
12563 || (r.e2e_status() == E2eStatus::NotConfigured
12564 && state.config.verify.e2e.is_empty()))
12565 }) || state
12566 .gate_repair_e2e
12567 .as_ref()
12568 .is_some_and(|e| e.head == head && e.passed)
12569 {
12570 return Ok(true);
12571 }
12572 if let Some(e) = &state.gate_repair_e2e
12573 && e.head == head
12574 && !e.passed
12575 && !e.outcomes.is_empty()
12576 && !verify_inconclusive(&e.outcomes)
12577 {
12578 state.status = RunStatus::Blocked;
12579 state.save()?;
12580 return Ok(false);
12581 }
12582 state.head_e2e = Some(crate::run::GateRepairE2e {
12583 head: head.clone(),
12584 passed: false,
12585 outcomes: Vec::new(),
12586 });
12587 state.save()?;
12588 let commands = state.config.verify.e2e.clone();
12589 let shell = state.config.shell();
12590 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12591 let cache_dir = state.config.cache_dir();
12592 let outcomes = if commands.is_empty() {
12593 Vec::new()
12594 } else {
12595 let (outcomes, _) = with_cache_lease(
12596 state,
12597 cache_dir.as_deref(),
12598 "e2e",
12599 "e2e",
12600 &winner.worktree,
12601 &head,
12602 timeout,
12603 "current head e2e",
12604 wait,
12605 |state, budget| async move {
12606 run_e2e_with_retry(
12607 state,
12608 &shell,
12609 &commands,
12610 &winner.worktree,
12611 budget,
12612 "current head e2e",
12613 )
12614 .await
12615 },
12616 )
12617 .await?;
12618 outcomes
12619 };
12620 let unresolved = verify_inconclusive(&outcomes);
12621 let passed = !unresolved
12622 && outcomes.iter().all(CommandOutcome::ok)
12623 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12624 if !outcomes.is_empty() && !unresolved {
12625 state.record_base_progress(&head);
12626 }
12627 state.head_e2e = Some(crate::run::GateRepairE2e {
12628 head: head.clone(),
12629 passed,
12630 outcomes,
12631 });
12632 if !passed && !unresolved {
12633 state.status = RunStatus::Blocked;
12634 state.event(
12635 "e2e",
12636 format!("verify.e2e failed on head {head}; nothing published"),
12637 );
12638 }
12639 state.save()?;
12640 Ok(passed)
12641}
12642
12643pub(crate) async fn check_gate_repair_e2e(
12646 state: &mut RunState,
12647 winner: &Candidate,
12648 wait: &crate::repo_lock::WaitPolicy,
12649) -> Result<GateFix> {
12650 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12651 return Ok(GateFix::Retry);
12652 }
12653 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12654 if state
12655 .gate_repair_e2e
12656 .as_ref()
12657 .is_some_and(|r| r.head == head && r.passed)
12658 {
12659 return Ok(GateFix::Retry);
12660 }
12661 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12662 head: head.clone(),
12663 passed: false,
12664 outcomes: Vec::new(),
12665 });
12666 state.save()?;
12667 let commands = state.config.verify.e2e.clone();
12668 let outcomes = if commands.is_empty() {
12669 Vec::new()
12670 } else {
12671 let shell = state.config.shell();
12672 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12673 let cache_dir = state.config.cache_dir();
12674 let (outcomes, _) = with_cache_lease(
12675 state,
12676 cache_dir.as_deref(),
12677 "e2e",
12678 "e2e",
12679 &winner.worktree,
12680 &head,
12681 timeout,
12682 "gate repair e2e",
12683 wait,
12684 |state, budget| async move {
12685 run_e2e_with_retry(
12686 state,
12687 &shell,
12688 &commands,
12689 &winner.worktree,
12690 budget,
12691 "gate repair e2e",
12692 )
12693 .await
12694 },
12695 )
12696 .await?;
12697 outcomes
12698 };
12699 let deferred = verify_inconclusive(&outcomes);
12700 let passed = outcomes.iter().all(CommandOutcome::ok)
12701 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12702 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12703 head,
12704 passed,
12705 outcomes,
12706 });
12707 state.save()?;
12708 if deferred {
12709 Ok(GateFix::Defer)
12710 } else if !passed {
12711 state.event(
12712 "gate",
12713 "verify.e2e failed after gate repair; nothing published",
12714 );
12715 Ok(GateFix::Stop)
12716 } else {
12717 Ok(GateFix::Retry)
12718 }
12719}
12720
12721pub(crate) async fn run_landing_gate(
12723 state: &mut RunState,
12724 winner: &Candidate,
12725 wait: &crate::repo_lock::WaitPolicy,
12726) -> Result<Vec<CommandOutcome>> {
12727 let shell = state.config.shell();
12728 let gate_commands = state.config.verify.gate.clone();
12729 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12730 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12731 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12732 Vec::new()
12733 } else {
12734 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12735 let cache_dir = state.config.cache_dir();
12736 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12737 let (outcomes, _) = with_cache_lease(
12738 state,
12739 cache_dir.as_deref(),
12740 "gate",
12741 "gate",
12742 &winner.worktree,
12743 &head,
12744 timeout,
12745 "final gate",
12746 wait,
12747 |state, budget| {
12748 let shell = shell.clone();
12749 let gate_commands = gate_commands.clone();
12750 let winner = winner.clone();
12751 async move {
12752 Runner::run_pre_gate(state, &winner).await;
12753 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12755 let (outcomes, timed_out_pids) = run_commands(
12756 state,
12757 "gate",
12758 "gate",
12759 0,
12760 &shell,
12761 &gate_commands,
12762 &winner.worktree,
12763 budget,
12764 )
12765 .await;
12766 (outcomes, false, timed_out_pids)
12767 }
12768 },
12769 )
12770 .await?;
12771 outcomes
12772 };
12773 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12774 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12775 let mut outcomes = outcomes;
12776 outcomes.push(CommandOutcome {
12777 command: "gate head stability".into(), code: None,
12778 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12779 duration_ms: 0, resource_blocked: false,
12780 });
12781 return Ok(outcomes);
12782 }
12783 if outcomes.is_empty() {
12784 state.event("gate", "no gate commands configured; not tested");
12789 }
12790 for o in &outcomes {
12791 state.event(
12792 "gate",
12793 format!(
12794 "`{}` -> {}",
12795 o.command,
12796 if o.ok() {
12797 "pass".to_owned()
12798 } else {
12799 format!(
12800 "FAIL ({:?})\n{}",
12801 o.code,
12802 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12803 )
12804 }
12805 ),
12806 );
12807 }
12808 Ok(outcomes)
12809}
12810
12811pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12817 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12818 ""
12819 } else {
12820 pr_title
12821 }
12822}
12823
12824pub fn landing_subject_source(state: &RunState) -> String {
12828 if is_review_run(state) {
12829 let winner = state.candidates.first().map_or('A', |c| c.label);
12830 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12831 }
12832 state.instruction.clone()
12833}
12834
12835fn summary_without_title(summary: &str) -> String {
12838 let mut lines = summary.trim().lines().peekable();
12839 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12840 lines.next();
12841 }
12842 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12843}
12844
12845#[cfg(test)]
12857fn pr_message(state: &RunState, winner: char) -> PrMessage {
12858 pr_message_with(state, winner, None)
12859}
12860
12861#[cfg(test)]
12864fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12865 let raw = pr_message_raw(
12866 state,
12867 winner,
12868 facts,
12869 crate::pr_outcome::FinalHead::Unreported,
12870 ChangeNote::NotAsked,
12871 );
12872 let id = crate::scrub::Identity::current();
12873 PrMessage {
12874 title: crate::scrub::scrub(&raw.title, &id),
12875 body: crate::scrub::scrub(&raw.body, &id),
12876 }
12877}
12878
12879#[derive(Debug, Clone, Copy)]
12881enum ChangeNote<'a> {
12882 NotAsked,
12884 Summary(&'a crate::run::ChangeSummary),
12886 Unavailable(&'a str),
12888}
12889
12890fn pr_message_raw(
12891 state: &RunState,
12892 winner: char,
12893 facts: Option<&BranchFacts>,
12894 head: crate::pr_outcome::FinalHead<'_>,
12895 change: ChangeNote<'_>,
12896) -> PrMessage {
12897 if !matches!(change, ChangeNote::NotAsked) {
12898 let (title, mut body) = match change {
12899 ChangeNote::Summary(c) => (
12900 c.title.clone(),
12901 format!("## Summary\n\n{}\n\n", c.body.trim()),
12902 ),
12903 ChangeNote::Unavailable(why) => (
12904 neutral_title(state, winner),
12905 format!(
12906 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12907 ),
12908 ),
12909 ChangeNote::NotAsked => unreachable!(),
12910 };
12911 body.push_str(&public_pr_evidence(state, head));
12912 body.push_str(&format!(
12913 "\n---\nmagi:run/{} magi:candidate-{}\n",
12914 state.id,
12915 winner.to_ascii_lowercase()
12916 ));
12917 return PrMessage { title, body };
12918 }
12919 pr_preview(state, winner, facts, head)
12920}
12921
12922fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12925 let mut body = crate::pr_outcome::review_rounds(state);
12926 body.push_str(&crate::pr_outcome::final_verification(state, head));
12927 let open = state.last_round_findings();
12928 if !open.is_empty() {
12929 body.push_str("## Open review findings\n\n");
12930 for f in open {
12931 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12932 }
12933 body.push('\n');
12934 }
12935 if let Some(c) = state.change_summary.as_ref()
12936 && head == crate::pr_outcome::FinalHead::At(&c.head)
12937 {
12938 body.push_str(&format!(
12939 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12940 c.base, c.head
12941 ));
12942 }
12943 body
12944}
12945
12946fn pr_preview(
12948 state: &RunState,
12949 winner: char,
12950 facts: Option<&BranchFacts>,
12951 head: crate::pr_outcome::FinalHead<'_>,
12952) -> PrMessage {
12953 let summary = state
12954 .candidates
12955 .iter()
12956 .find(|c| c.label == winner)
12957 .map(|c| c.summary.as_str())
12958 .unwrap_or_default();
12959 let review = is_review_run(state);
12962 let title = if review {
12963 facts
12964 .and_then(|f| {
12965 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12966 })
12967 .or_else(|| review_title(state))
12968 .or_else(|| {
12969 state
12970 .candidates
12971 .iter()
12972 .find(|c| c.label == winner)
12973 .filter(|c| !c.branch.starts_with("magi/"))
12974 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12975 })
12976 .unwrap_or_else(|| neutral_title(state, winner))
12977 } else {
12978 summary_title(summary).unwrap_or_else(|| {
12979 english_title(
12980 &state.instruction,
12981 PR_TITLE_MAX,
12982 state.config.graph.github_english(),
12983 )
12984 .unwrap_or_else(|| neutral_title(state, winner))
12985 })
12986 };
12987
12988 let mut body = String::new();
12989 let what = summary_without_title(summary);
12990 if !what.is_empty() {
12991 body.push_str("## Summary\n\n");
12992 body.push_str(&what);
12993 body.push_str("\n\n");
12994 }
12995
12996 body.push_str(&crate::pr_outcome::review_rounds(state));
12998 body.push_str(&crate::pr_outcome::final_verification(state, head));
12999 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13000
13001 let open = state.open_findings();
13002 if !open.is_empty() {
13003 body.push_str("## Open review findings\n\n");
13004 for f in &open {
13005 body.push_str(&format!(
13006 "- `{}` [{:?}] {}\n",
13007 f.id, f.severity, "Context retained in local review records."
13008 ));
13009 }
13010 body.push('\n');
13011 }
13012
13013 if let Some(fix) = fix
13014 && !fix.rejected.is_empty()
13015 {
13016 body.push_str("## Declined by the fixer\n\n");
13017 for r in &fix.rejected {
13018 body.push_str(&format!(
13019 "- `{}`: {}\n",
13020 r.id, "Reason retained in local review records."
13021 ));
13022 }
13023 body.push('\n');
13024 }
13025
13026 if review {
13027 body.push_str("## Commits under review\n\n");
13029 if let Some(facts) = facts {
13030 let mut left = COMMITS_MAX_CHARS;
13031 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13032 let mut entry = format!("- {}\n", markdown_safe(subject));
13033 for l in markdown_safe(text).lines() {
13034 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13035 }
13036 if left == 0 {
13037 body.push_str(&format!(
13038 "- ... {} more commit(s)\n",
13039 facts.commits.len() - i
13040 ));
13041 break;
13042 }
13043 if entry.len() > left {
13044 let mut end = left;
13047 while !entry.is_char_boundary(end) {
13048 end -= 1;
13049 }
13050 entry.truncate(end);
13051 entry.push_str("\n ... (truncated)\n");
13052 left = 0;
13053 } else {
13054 left -= entry.len();
13055 }
13056 body.push_str(&entry);
13057 }
13058 if !facts.stat.trim().is_empty() {
13059 body.push_str(&format!(
13060 "\n## Diff stat\n\n```\n{}\n```\n",
13061 markdown_safe(facts.stat.trim())
13062 ));
13063 }
13064 } else {
13065 match &state.reviewed_commits {
13066 Some(subjects) => {
13067 for s in subjects {
13068 body.push_str(&format!("- {}\n", s.trim()));
13069 }
13070 }
13071 None => {
13072 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13075 body.push_str(rest.trim());
13076 body.push('\n');
13077 }
13078 }
13079 }
13080 } else {
13081 let task = state.instruction.trim();
13082 let task = if task.is_empty() {
13083 "(empty task)"
13084 } else {
13085 task
13086 };
13087 body.push_str(&format!(
13088 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13089 task.replace("</details>", "</details>")
13090 ));
13091 }
13092
13093 body.push_str(&format!(
13094 "\n---\nmagi:run/{} magi:candidate-{}\n",
13095 state.id,
13096 winner.to_ascii_lowercase()
13097 ));
13098
13099 PrMessage { title, body }
13100}
13101
13102fn seeded_instruction(state: &RunState) -> String {
13105 let mut inputs = Vec::new();
13106 if let Some(start) = &state.sources.start_from {
13107 inputs.push(format!(
13108 "Start from `{}` at `{}` (including ancestry).",
13109 start.token, start.sha
13110 ));
13111 }
13112 for pick in &state.sources.pick_commits {
13113 inputs.push(format!(
13114 "Pick `{}` at `{}` (single commit).",
13115 pick.token, pick.sha
13116 ));
13117 }
13118 if inputs.is_empty() {
13119 state.instruction.clone()
13120 } else {
13121 format!(
13122 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13123 state.instruction,
13124 inputs.join("\n")
13125 )
13126 }
13127}
13128
13129async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13133 let base = &state.base_branch;
13134 let mut against = base.clone();
13135 if mode == MergeMode::Pr {
13136 let remote = &state.config.merge.remote;
13140 let tracking = format!("{remote}/{base}");
13141 let fetched = git::fetch(repo, remote, base).await;
13142 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13143 against = tracking;
13144 } else {
13145 return false;
13146 }
13147 }
13148 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13149}
13150
13151fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13154 let mut detail = format!(
13155 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13156 nothing to open a pull request for"
13157 );
13158 detail.push_str(
13159 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13160 );
13161 detail
13162}
13163
13164#[derive(Debug, PartialEq, Eq)]
13167enum PrPlan {
13168 Create,
13169 Adopt { url: String, title: String },
13170 Stop(String),
13171}
13172
13173fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13176 match found {
13177 Ok(land::OpenPr::None) => PrPlan::Create,
13178 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13179 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13180 "several open pull requests exist for this branch, not picking one: {}",
13181 urls.join(" ")
13182 )),
13183 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13184 }
13185}
13186
13187pub(crate) async fn gh_pr_create(
13189 cwd: &Path,
13190 base: &str,
13191 head: &str,
13192 title: &str,
13193 body: &str,
13194) -> Result<String> {
13195 let out = tokio::process::Command::new("gh")
13196 .args([
13197 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13198 ])
13199 .current_dir(cwd)
13200 .quiet()
13201 .stdin(std::process::Stdio::null())
13202 .output()
13203 .await
13204 .context("spawn gh")?;
13205 if out.status.success() {
13206 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13207 } else {
13208 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13209 }
13210}
13211
13212async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13214 match std::fs::symlink_metadata(path) {
13215 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13216 git::worktree_remove_clean(repo, path).await?;
13219 return Ok(());
13220 }
13221 Err(e) => return Err(e.into()),
13222 Ok(_) => {}
13223 }
13224 if !git::is_clean(path).await? {
13225 bail!(
13226 "worktree has uncommitted work; refusing removal: {}",
13227 path.display()
13228 );
13229 }
13230 if !git::worktree_remove_clean(repo, path).await? {
13231 bail!("worktree removal refused: {}", path.display());
13232 }
13233 Ok(())
13234}
13235
13236pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13245 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13246 bail!("refusing fold: {reason}");
13247 }
13248 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13249 bail!("candidate preservation incomplete; refusing cleanup");
13250 }
13251 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13255 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13256 let _driver = crate::driver::maintenance(home, &state.id)?;
13257 let mut output_guards = Vec::new();
13258 for entry in std::fs::read_dir(state.worktree_root())
13259 .into_iter()
13260 .flatten()
13261 .flatten()
13262 {
13263 if entry.path().join(".git").is_file() {
13264 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13265 }
13266 }
13267 let repo = state.repo.clone();
13268 let wait = repository_wait(state, None);
13271 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13272 Ok(lease) => lease,
13273 Err(e) => {
13274 state.save_under(home)?;
13275 return Err(e);
13276 }
13277 };
13278 let _source_outputs = if state.status == RunStatus::Merged
13281 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13282 {
13283 verify_merged_fold_sources(state, drop_winner, home).await?
13284 } else {
13285 Vec::new()
13286 };
13287 let root = state.worktree_root();
13288 let winner = state.tally.as_ref().map(|t| t.winner);
13289 let mut removed = Vec::new();
13290
13291 for i in 0..state.candidates.len() {
13292 let c = state.candidates[i].clone();
13293 let is_winner = Some(c.label) == winner;
13294 if is_winner && !drop_winner {
13295 continue;
13296 }
13297 let existed = c.worktree.exists();
13298 remove_preserved_worktree(&repo, &c.worktree).await?;
13301 if existed {
13302 removed.push(c.worktree.to_string_lossy().into_owned());
13303 }
13304 let handed_over = state.released_branches.contains(&c.branch);
13307 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13308 if !git::branch_delete(&repo, &c.branch).await? {
13309 bail!("candidate branch removal refused");
13310 }
13311 removed.push(c.branch.clone());
13312 }
13313 state.candidates[i].folded = true;
13314 }
13315
13316 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13317 let path = name.path();
13318 let keep = !drop_winner
13319 && winner.is_some_and(|w| {
13320 path.file_name()
13321 .is_some_and(|n| n == format!("cand-{w}").as_str())
13322 });
13323 if keep {
13324 continue;
13325 }
13326 remove_preserved_worktree(&repo, &path).await?;
13327 removed.push(path.to_string_lossy().into_owned());
13328 }
13329
13330 remove_if_empty(&root);
13339
13340 if state.enabled_worktree_config && drop_winner {
13341 git::release_worktree_config(&repo).await.ok();
13345 state.enabled_worktree_config = false;
13346 }
13347 state.save_under(home)?;
13348 Ok(removed)
13349}
13350
13351async fn verify_merged_fold_sources(
13352 state: &RunState,
13353 drop_winner: bool,
13354 home: &Path,
13355) -> Result<Vec<std::fs::File>> {
13356 let winner = state.tally.as_ref().map(|t| t.winner);
13357 let mut heads = BTreeSet::new();
13358 for candidate in &state.candidates {
13359 if Some(candidate.label) == winner && !drop_winner {
13360 continue;
13361 }
13362 anyhow::ensure!(
13363 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13364 "candidate recovery source is outside the inventoried run root: {}",
13365 candidate.worktree.display()
13366 );
13367 match std::fs::symlink_metadata(&candidate.worktree) {
13368 Ok(_) => {}
13369 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13370 Err(error) => return Err(error.into()),
13371 }
13372 if !state.released_branches.contains(&candidate.branch)
13373 && git::branch_exists(&state.repo, &candidate.branch).await?
13374 {
13375 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13376 }
13377 }
13378 let mut cargo_guards = Vec::new();
13379 let entries = match std::fs::read_dir(state.worktree_root()) {
13380 Ok(entries) => Some(entries),
13381 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13382 Err(error) => return Err(error.into()),
13383 };
13384 for entry in entries.into_iter().flatten() {
13385 let path = entry?.path();
13386 let metadata = std::fs::symlink_metadata(&path)?;
13387 anyhow::ensure!(
13388 metadata.is_dir() && !metadata.file_type().is_symlink(),
13389 "non-directory or symlink recovery source retained: {}",
13390 path.display()
13391 );
13392 if !drop_winner
13393 && winner.is_some_and(|w| {
13394 path.file_name()
13395 .is_some_and(|name| name == format!("cand-{w}").as_str())
13396 })
13397 {
13398 continue;
13399 }
13400 anyhow::ensure!(
13401 path.join(".git").is_file(),
13402 "unregistered recovery source retained: {}",
13403 path.display()
13404 );
13405 anyhow::ensure!(
13406 git::git(
13407 &path,
13408 &[
13409 "status",
13410 "--porcelain",
13411 "--untracked-files=all",
13412 "--ignored",
13413 "--",
13414 ".",
13415 ":(exclude)target"
13416 ]
13417 )
13418 .await?
13419 .is_empty(),
13420 "source changes or ignored recovery files retained: {}",
13421 path.display()
13422 );
13423 match std::fs::symlink_metadata(path.join("target")) {
13424 Ok(_) => {
13425 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13426 }
13427 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13428 Err(error) => return Err(error.into()),
13429 }
13430 for checkpoint in [
13431 "rebase-merge",
13432 "rebase-apply",
13433 "sequencer",
13434 "CHERRY_PICK_HEAD",
13435 "MERGE_HEAD",
13436 "REBASE_HEAD",
13437 "REVERT_HEAD",
13438 "BISECT_LOG",
13439 "AUTO_MERGE",
13440 "MERGE_MSG",
13441 ] {
13442 let location = git::git(
13443 &path,
13444 &[
13445 "rev-parse",
13446 "--path-format=absolute",
13447 "--git-path",
13448 checkpoint,
13449 ],
13450 )
13451 .await?;
13452 anyhow::ensure!(
13453 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13454 if e.kind() == std::io::ErrorKind::NotFound),
13455 "Git recovery checkpoint retained: {checkpoint}"
13456 );
13457 }
13458 heads.insert(git::rev_parse(&path, "HEAD").await?);
13459 }
13460 crate::fold_evidence::preserve(state, home, &heads).await?;
13461
13462 Ok(cargo_guards)
13463}
13464
13465fn remove_if_empty(dir: &Path) {
13476 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13477 std::fs::remove_dir(dir).ok();
13478 }
13479}
13480
13481pub fn worst_open(state: &RunState) -> Option<Severity> {
13483 state
13484 .reviews
13485 .last()?
13486 .reviews
13487 .iter()
13488 .flat_map(|r| r.findings.iter())
13489 .map(|f| f.severity)
13490 .max()
13491}
13492
13493async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13497 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13498 git::acquire_worktree_config(repo).await?;
13499 state.enabled_worktree_config = true;
13500 }
13501 Ok(())
13502}
13503
13504#[cfg(test)]
13505mod tests {
13506 #[tokio::test]
13507 async fn launcher_base_fetch_requires_remote_success() {
13508 let tmp = tempfile::tempdir().unwrap();
13509 let repo = tmp.path().join("repo");
13510 let remote = tmp.path().join("remote");
13511 std::fs::create_dir(&repo).unwrap();
13512 std::fs::create_dir(&remote).unwrap();
13513 git::git(&remote, &["init", "--bare", "-b", "main"])
13514 .await
13515 .unwrap();
13516 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13517 git::git(
13518 &repo,
13519 &[
13520 "-c",
13521 "user.name=Fixture",
13522 "-c",
13523 "user.email=fixture@example.invalid",
13524 "commit",
13525 "--allow-empty",
13526 "-m",
13527 "fixture",
13528 ],
13529 )
13530 .await
13531 .unwrap();
13532 git::git(
13533 &repo,
13534 &["remote", "add", "origin", remote.to_str().unwrap()],
13535 )
13536 .await
13537 .unwrap();
13538 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13539 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13540 assert_eq!(
13541 resolve_base(&repo, "main", "origin", None)
13542 .await
13543 .unwrap()
13544 .commit,
13545 head
13546 );
13547 #[cfg(unix)]
13548 {
13549 let policy: crate::config::Coordination = toml::from_str(
13550 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13551 ).unwrap();
13552 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13553 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13554 let held = crate::repo_lock::SeatLease::operation_with(
13555 &repo,
13556 Some(&policy),
13557 &crate::repo_lock::WaitPolicy::default(),
13558 )
13559 .await
13560 .unwrap();
13561 assert!(held.is_some());
13562 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13563 tokio::pin!(fetch);
13564 assert!(
13565 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13566 .await
13567 .is_err()
13568 );
13569 assert!(
13570 !repo.join(".git/FETCH_HEAD").exists(),
13571 "no fetch before admission"
13572 );
13573 drop(held);
13574 let fetched = fetch.await.unwrap();
13575 assert_eq!(fetched.commit, head);
13576 assert_eq!(fetched.remote, "origin");
13577 assert_eq!(fetched.branch, "main");
13578 }
13579 git::git(
13581 &repo,
13582 &[
13583 "remote",
13584 "set-url",
13585 "origin",
13586 tmp.path().join("missing").to_str().unwrap(),
13587 ],
13588 )
13589 .await
13590 .unwrap();
13591 let error = resolve_base(&repo, "main", "origin", None)
13592 .await
13593 .unwrap_err()
13594 .to_string();
13595 assert!(error.contains("cannot read origin/main"));
13596 assert!(error.contains("refusing to branch off"));
13597 }
13598
13599 #[test]
13600 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13601 let leaked = vec!["chore(deps): update a crate".to_owned()];
13602 let own = "fix(daemon): apply a chosen action";
13603 assert!(should_retitle("", own, &leaked));
13604 assert!(should_retitle(
13605 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13606 own,
13607 &leaked
13608 ));
13609 assert!(should_retitle(
13610 "chore: land candidate A of run 1",
13611 own,
13612 &leaked
13613 ));
13614 assert!(should_retitle(
13615 "magi: candidate A (uncommitted work)",
13616 own,
13617 &leaked
13618 ));
13619 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13620 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13621 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13622 }
13623
13624 #[test]
13625 fn pr_merge_plan_creates_adopts_or_stops() {
13626 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13627 assert_eq!(
13628 pr_merge_plan(Ok(land::OpenPr::One {
13629 url: "u".into(),
13630 title: "t".into()
13631 })),
13632 PrPlan::Adopt {
13633 url: "u".into(),
13634 title: "t".into()
13635 }
13636 );
13637 let PrPlan::Stop(many) =
13638 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13639 else {
13640 panic!("many must stop");
13641 };
13642 assert!(many.contains('a') && many.contains('b'));
13643 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13644 panic!("a failed lookup must stop");
13645 };
13646 assert!(err.contains("bad token"));
13647 }
13648
13649 use super::*;
13650 use crate::run::GateStatus;
13651 use std::collections::BTreeMap;
13652 use std::time::Duration;
13653
13654 fn conductor() -> AgentSpec {
13655 AgentSpec {
13656 id: "conductor".to_owned(),
13657 kind: crate::config::AgentKind::Command,
13658 model: None,
13659 command: vec!["true".to_owned()],
13660 extra_args: Vec::new(),
13661 env: BTreeMap::new(),
13662 prompt_delivery: None,
13663 max_turn_secs: None,
13664 }
13665 }
13666
13667 fn spec(id: &str) -> AgentSpec {
13668 AgentSpec {
13669 id: id.to_owned(),
13670 kind: crate::config::AgentKind::Command,
13671 model: None,
13672 command: vec!["true".to_owned()],
13673 extra_args: Vec::new(),
13674 env: BTreeMap::new(),
13675 prompt_delivery: None,
13676 max_turn_secs: None,
13677 }
13678 }
13679
13680 fn ids(xs: &[&str]) -> BTreeSet<String> {
13681 xs.iter().map(|s| (*s).to_owned()).collect()
13682 }
13683
13684 #[test]
13685 fn pick_successor_skips_an_agent_another_seat_holds() {
13686 let roster = [spec("a"), spec("b"), spec("c")];
13688 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13689 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13690 }
13691
13692 #[test]
13693 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13694 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13696 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13697 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13698 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13699 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13700 }
13701
13702 #[test]
13703 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13704 let roster = [spec("a"), spec("b")];
13705 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13706 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13707 let carried = ids(&["b"]);
13708 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13709 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13710 }
13711
13712 #[test]
13713 fn pick_successor_returns_none_without_any_untried_successor() {
13714 let roster = [spec("a"), spec("b")];
13715 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13716 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13717 let carried = ids(&["a"]);
13718 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13719 }
13720
13721 #[test]
13722 fn pick_successor_rescue_avoids_another_seats_occupant() {
13723 let roster = [spec("a"), spec("b"), spec("c")];
13725 let carried = ids(&["b", "c"]);
13726 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13727 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13728 }
13729
13730 #[test]
13731 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13732 let roster = [spec("a"), spec("b"), spec("c")];
13733 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13734 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13735 }
13736
13737 #[test]
13738 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13739 let roster = [spec("a"), spec("b"), spec("c")];
13740 let failed = ids(&["a", "b", "c"]);
13741 let mut tried = ids(&["b"]);
13743 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13744 assert_eq!(first.id, "a");
13745 tried.insert(first.id.clone());
13746 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13747 assert_eq!(second.id, "c");
13748 tried.insert(second.id.clone());
13749 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13750 }
13751
13752 #[test]
13753 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13754 let roster = [spec("a"), spec("b")];
13755 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13756 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13757 }
13758
13759 #[test]
13760 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13761 let roster = [spec("a"), spec("b"), spec("c")];
13762 let h = SeatHistory {
13763 failed: ids(&["a"]),
13764 last_ok: Some("b".to_owned()),
13765 last_fail: None,
13766 };
13767 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13768 let gone = SeatHistory {
13770 last_ok: Some("zzz".to_owned()),
13771 ..h.clone()
13772 };
13773 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13774 let failed = SeatHistory {
13775 failed: ids(&["a", "b"]),
13776 last_ok: Some("b".to_owned()),
13777 last_fail: None,
13778 };
13779 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13780 }
13781
13782 #[test]
13783 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13784 let first = SeatState::new("review-1", "alpha", 7);
13785 let next = handover_seat("review-1", "gamma", 7);
13786 assert_ne!(first.claude_session, next.claude_session);
13787 }
13788
13789 #[test]
13790 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13791 let mut state = state_with_summary("x", "y");
13792 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13793 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13794 assert_ne!(a.claude_session, b.claude_session);
13795 }
13796
13797 #[test]
13798 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13799 let roster = [spec("a"), spec("b")];
13800 let h = SeatHistory {
13801 failed: ids(&["a", "b"]),
13802 ..SeatHistory::default()
13803 };
13804 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13805 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13806 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13807 }
13808
13809 #[test]
13816 fn failure_signature_ignores_numbers_and_paths() {
13817 assert_eq!(
13818 failure_signature("exited with Some(2) and no usable output"),
13819 failure_signature("exited with Some(137) and no usable output")
13820 );
13821 assert_eq!(
13822 failure_signature("process exited with no usable output (exit code 2)"),
13823 failure_signature("process exited with no usable output (exit code 137)")
13824 );
13825 for (old, new) in [
13826 (
13827 "exited with Some(2) and no usable output",
13828 "process exited with no usable output (exit code 2)",
13829 ),
13830 (
13831 "exited with None and no usable output",
13832 "process exited with no usable output (exit code unavailable)",
13833 ),
13834 ] {
13835 let previous = FailClass::Other(failure_signature(old));
13836 let current = FailClass::Other(failure_signature(new));
13837 assert_eq!(previous, current, "resumption retains the failure class");
13838 assert!(!should_hand_over(Some(&previous), ¤t));
13839 }
13840 assert_eq!(
13841 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13842 failure_signature("cannot open /var/x.txt: denied")
13843 );
13844 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13845 }
13846
13847 #[test]
13848 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13849 let other = FailClass::Other("x".into());
13850 assert!(should_hand_over(None, &FailClass::Quota));
13851 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13852 assert!(should_hand_over(
13853 Some(&FailClass::Timeout),
13854 &FailClass::Timeout
13855 ));
13856 assert!(should_hand_over(None, &other));
13857 assert!(!should_hand_over(Some(&other), &other));
13858 assert!(should_hand_over(
13859 Some(&other),
13860 &FailClass::Other("y".into())
13861 ));
13862 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13864 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13865 }
13866
13867 #[test]
13868 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13869 let a = SeatState::new("judge-1", "alpha", 7);
13870 let b = handover_seat("judge-1", "beta", 7);
13871 assert_ne!(a.claude_session, b.claude_session);
13872 assert_eq!(b.turns, 0);
13873 }
13874
13875 #[test]
13876 fn a_timeout_is_classified_apart_from_other_failures() {
13877 assert_eq!(
13878 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13879 Some(FailClass::Timeout)
13880 );
13881 assert!(matches!(
13882 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13883 Some(FailClass::Other(_))
13884 ));
13885 }
13886
13887 #[test]
13888 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13889 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13890 let tried = BTreeSet::from(["beta".to_owned()]);
13891 let next = next_untried_in_roster(&roster, 1, &tried);
13894 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13895 }
13896
13897 #[test]
13898 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13899 let roster = vec![spec("alpha"), spec("beta")];
13900 let tried = BTreeSet::from(["beta".to_owned()]);
13901 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13905 }
13906
13907 #[test]
13908 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13909 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13910 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13911 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13915 }
13916
13917 #[test]
13918 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13919 let roster = vec![spec("a"), spec("a"), spec("b")];
13920 let tried = BTreeSet::from(["a".to_owned()]);
13921 let next = next_untried_in_roster(&roster, 0, &tried);
13922 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13923 }
13924
13925 #[test]
13926 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13927 let roster = vec![spec("a"), spec("b")];
13928 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13929 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13930 }
13931
13932 #[test]
13933 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13934 let dir = tempfile::tempdir().unwrap();
13935 let bay = dir.path().join("ffff");
13936
13937 remove_if_empty(&bay);
13939 assert!(!bay.exists());
13940
13941 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13944 remove_if_empty(&bay);
13945 assert!(bay.exists(), "non-empty directory must survive");
13946
13947 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13949 remove_if_empty(&bay);
13950 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13951 }
13952
13953 #[test]
13962 fn a_full_panel_that_found_nothing_is_clean() {
13963 assert!(round_is_clean(
13964 0,
13965 true,
13966 2,
13967 2,
13968 0,
13969 IncompleteReviewPolicy::Block
13970 ));
13971 }
13972
13973 #[test]
13974 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13975 assert!(!round_is_clean(
13976 0,
13977 true,
13978 1,
13979 2,
13980 0,
13981 IncompleteReviewPolicy::Block
13982 ));
13983 }
13984
13985 #[test]
13986 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13987 assert!(!round_is_clean(
13988 1,
13989 true,
13990 1,
13991 2,
13992 0,
13993 IncompleteReviewPolicy::Warn
13994 ));
13995 }
13996
13997 #[test]
13998 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13999 assert!(round_is_clean(
14000 0,
14001 true,
14002 1,
14003 2,
14004 0,
14005 IncompleteReviewPolicy::Warn
14006 ));
14007 }
14008
14009 #[test]
14010 fn a_full_panel_with_an_open_finding_is_not_clean() {
14011 assert!(!round_is_clean(
14012 1,
14013 true,
14014 2,
14015 2,
14016 0,
14017 IncompleteReviewPolicy::Block
14018 ));
14019 }
14020
14021 #[test]
14022 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14023 assert!(!round_is_clean(
14024 0,
14025 false,
14026 2,
14027 2,
14028 0,
14029 IncompleteReviewPolicy::Block
14030 ));
14031 }
14032
14033 #[test]
14040 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14041 assert!(round_is_clean(
14044 0,
14045 true,
14046 1,
14047 2,
14048 1,
14049 IncompleteReviewPolicy::Block
14050 ));
14051 }
14052
14053 #[test]
14054 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14055 assert!(!round_is_clean(
14058 0,
14059 true,
14060 1,
14061 2,
14062 0,
14063 IncompleteReviewPolicy::Block
14064 ));
14065 }
14066
14067 #[test]
14068 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14069 assert!(!round_is_clean(
14070 1,
14071 true,
14072 1,
14073 2,
14074 1,
14075 IncompleteReviewPolicy::Block
14076 ));
14077 assert!(!round_is_clean(
14078 0,
14079 false,
14080 1,
14081 2,
14082 1,
14083 IncompleteReviewPolicy::Block
14084 ));
14085 }
14086
14087 #[test]
14088 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14089 assert!(!round_is_clean(
14093 0,
14094 true,
14095 0,
14096 2,
14097 2,
14098 IncompleteReviewPolicy::Block
14099 ));
14100 }
14101
14102 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14103 CommandOutcome {
14104 command: "test".to_owned(),
14105 code,
14106 output_tail: String::new(),
14107 duration_ms: 0,
14108 resource_blocked,
14109 }
14110 }
14111
14112 #[cfg(unix)]
14113 #[tokio::test]
14114 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14115 ask_test_home();
14116 let temp = tempfile::tempdir().unwrap();
14117 let mut runner = runner_at(RunStatus::Gating);
14118 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14119 let commands = vec![
14120 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14121 "touch must-not-run".to_owned(),
14122 ];
14123 let (outcomes, retried, pids) = run_e2e_with_retry(
14124 &mut runner.state,
14125 &shell,
14126 &commands,
14127 temp.path(),
14128 Duration::from_secs(5),
14129 "fixture",
14130 )
14131 .await;
14132 assert_eq!(outcomes.len(), 1);
14133 assert!(outcomes[0].resource_blocked);
14134 assert!(!outcomes[0].build_failed());
14135 assert!(!retried);
14136 assert!(pids.is_empty());
14137 assert!(!temp.path().join("must-not-run").exists());
14138 assert_eq!(
14139 runner.state.resource_block.as_ref().unwrap().kind,
14140 "incremental cache admission"
14141 );
14142 }
14143
14144 #[test]
14145 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14146 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14147 assert!(
14148 !verify_inconclusive(&[outcome(Some(1), false)]),
14149 "an ordinary failure is still evidence about the patch"
14150 );
14151 assert!(verify_inconclusive(&[outcome(None, true)]));
14152 assert!(
14153 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14154 "one inconclusive outcome taints the whole batch"
14155 );
14156 assert!(!verify_inconclusive(&[]));
14157 }
14158
14159 #[tokio::test]
14160 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14161 let calls = std::sync::atomic::AtomicUsize::new(0);
14165 let started = Instant::now();
14166 wait_for_pids_with(
14167 &[123],
14168 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14169 Duration::from_millis(5),
14170 Duration::from_secs(5),
14171 )
14172 .await;
14173 assert!(
14174 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14175 "must keep checking rather than deciding on the first answer"
14176 );
14177 assert!(
14178 started.elapsed() < Duration::from_secs(1),
14179 "must return the moment it is confirmed dead, not wait out the ceiling"
14180 );
14181 }
14182
14183 #[tokio::test]
14184 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14185 let started = Instant::now();
14186 wait_for_pids_with(
14187 &[123],
14188 |_| true, Duration::from_millis(5),
14190 Duration::from_millis(30),
14191 )
14192 .await;
14193 let elapsed = started.elapsed();
14194 assert!(
14195 elapsed >= Duration::from_millis(30),
14196 "must not give up before its own ceiling: {elapsed:?}"
14197 );
14198 assert!(
14199 elapsed < Duration::from_secs(1),
14200 "must not wait past its own ceiling either: {elapsed:?}"
14201 );
14202 }
14203
14204 #[tokio::test]
14205 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14206 let started = Instant::now();
14207 wait_for_pids_with(
14208 &[],
14209 |_| true,
14210 Duration::from_secs(5),
14211 Duration::from_secs(5),
14212 )
14213 .await;
14214 assert!(
14215 started.elapsed() < Duration::from_millis(200),
14216 "an empty pid list has nothing to confirm"
14217 );
14218 }
14219
14220 fn review_round(
14226 clean: bool,
14227 blocking: usize,
14228 answered: usize,
14229 expected: usize,
14230 progressed: bool,
14231 e2e_ok: bool,
14232 ) -> ReviewRound {
14233 ReviewRound {
14234 round: 1,
14235 head: "h".to_owned(),
14236 verified_head: None,
14237 verified_at: None,
14238 reviews: Vec::new(),
14239 e2e: vec![CommandOutcome {
14240 command: "test".to_owned(),
14241 code: Some(if e2e_ok { 0 } else { 1 }),
14242 output_tail: String::new(),
14243 duration_ms: 0,
14244 resource_blocked: false,
14245 }],
14246 verify_retried: false,
14247 e2e_deferred: false,
14248 e2e_defer_reason: None,
14249 fix: None,
14250 blocking,
14251 answered,
14252 expected,
14253 clean,
14254 progressed,
14255 vote_split: false,
14256 reconsideration: Vec::new(),
14257 verdict: None,
14258 }
14259 }
14260
14261 #[test]
14262 fn review_conclusion_is_none_when_nothing_has_run() {
14263 assert_eq!(review_conclusion(&[], 3), None);
14264 }
14265
14266 #[test]
14267 fn review_conclusion_is_none_while_rounds_remain() {
14268 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14269 assert_eq!(review_conclusion(&rounds, 3), None);
14270 }
14271
14272 #[test]
14273 fn review_conclusion_is_gating_once_a_round_is_clean() {
14274 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14275 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14276 }
14277
14278 #[test]
14279 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14280 let rounds = vec![
14281 review_round(false, 1, 2, 2, true, true),
14282 review_round(false, 1, 2, 2, true, true),
14283 ];
14284 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14285 }
14286
14287 #[test]
14288 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14289 let rounds = vec![
14290 review_round(false, 1, 2, 2, true, true),
14291 review_round(false, 1, 2, 2, true, false),
14292 ];
14293 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14294 }
14295
14296 #[test]
14297 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14298 let mut blocked = review_round(false, 1, 2, 2, true, false);
14305 blocked.e2e[0].resource_blocked = true;
14306 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14307 assert_eq!(review_conclusion(&rounds, 2), None);
14308 }
14309
14310 #[test]
14311 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14312 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14314 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14315 }
14316
14317 #[test]
14318 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14319 let rounds = vec![
14320 review_round(false, 1, 2, 2, false, true),
14321 review_round(false, 1, 2, 2, false, true),
14322 ];
14323 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14324 }
14325
14326 fn secs(n: u64) -> Duration {
14327 Duration::from_secs(n)
14328 }
14329
14330 #[tokio::test]
14333 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14334 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14335 if std::env::var_os(CHILD).is_none() {
14336 let output = std::process::Command::new(std::env::current_exe().unwrap())
14340 .args([
14341 "--exact",
14342 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14343 "--nocapture",
14344 ])
14345 .env(CHILD, "1")
14346 .env_remove("MAGI_WORKER_HOME")
14347 .env_remove("MAGI_WORKER_ID")
14348 .output()
14349 .unwrap();
14350 assert!(
14351 output.status.success()
14352 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14353 "{}\n{}",
14354 String::from_utf8_lossy(&output.stdout),
14355 String::from_utf8_lossy(&output.stderr)
14356 );
14357 return;
14358 }
14359 ask_test_home();
14360 let tmp = tempfile::tempdir().unwrap();
14361 let repo = tmp.path().join("repo");
14362 std::fs::create_dir_all(&repo).unwrap();
14363 init_repo(&repo);
14364 let g = |args: &[&str]| {
14365 let out = std::process::Command::new("git")
14366 .args(args)
14367 .current_dir(&repo)
14368 .quiet()
14369 .output()
14370 .unwrap();
14371 assert!(
14372 out.status.success(),
14373 "{}",
14374 String::from_utf8_lossy(&out.stderr)
14375 );
14376 };
14377 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14378 g(&["checkout", "-b", "historical"]);
14379 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14380 g(&["add", "."]);
14381 g(&["commit", "-m", "old alternative"]);
14382 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14383 g(&["checkout", "-b", "magi/task-selected", &base]);
14384 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14385 g(&["add", "."]);
14386 g(&["commit", "-m", "measurement infrastructure"]);
14387 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14388 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14389 g(&["add", "."]);
14390 g(&["commit", "-m", "dependent implementation"]);
14391 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14392 g(&["checkout", "main"]);
14393 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14394 g(&["add", "."]);
14395 g(&["commit", "-m", "unrelated base change"]);
14396 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14397 let origin = tmp.path().join("origin.git");
14398 g(&[
14399 "clone",
14400 "--bare",
14401 repo.to_str().unwrap(),
14402 origin.to_str().unwrap(),
14403 ]);
14404 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14405 g(&["fetch", "origin"]);
14406 let selection = crate::pr_recovery::Selection {
14407 url: "https://example.test/project/pull/1".into(),
14408 branch: "magi/task-selected".into(),
14409 head: head.clone(),
14410 source_head: None,
14411 source_run: None,
14412 source_proof: None,
14413 disposition: Some(crate::pr_recovery::Disposition::Review),
14414 instruction: "preserve measured behavior".into(),
14415 approved_at: jiff::Timestamp::now(),
14416 };
14417 let mut template = runner_at(RunStatus::Reviewing);
14418 template.state.repo = repo.clone();
14419 template.state.base_commit = current.clone();
14420 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14421 template.state.origin = Some(crate::run::Origin::queue("selected"));
14422 template.state.pr_head = Some(selection.branch.clone());
14423 template.state.pr_recovery = Some(selection.clone());
14424 let mut runner =
14425 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14426 .await
14427 .unwrap();
14428 let wt = runner.state.winner().unwrap().worktree.clone();
14429 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14430 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14431 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14432 assert!(runner.state.seeds.is_empty());
14433 assert_eq!(runner.state.instruction, "preserve measured behavior");
14434 runner.sync_to_base().await.unwrap();
14435 assert_eq!(
14436 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14437 "prerequisite"
14438 );
14439 assert_eq!(
14440 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14441 "measured implementation"
14442 );
14443 assert_eq!(
14444 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14445 "current main change"
14446 );
14447 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14448 assert_eq!(
14449 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14450 head
14451 );
14452
14453 let mut partial = runner_at(RunStatus::Reviewing);
14455 partial.state.repo = repo.clone();
14456 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14457 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14458 run: Some("exit 1".into()),
14459 ..Default::default()
14460 }];
14461 let selection = runner.state.pr_recovery.clone().unwrap();
14462 partial.state.pr_recovery = Some(selection.clone());
14463 partial.state.pr_head = Some(selection.branch.clone());
14464 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14465 let id = partial.state.id.clone();
14466 let error = match Runner::open_selected(
14467 &repo,
14468 partial.state,
14469 partial.roles,
14470 1,
14471 runner.state.base_commit.clone(),
14472 selection,
14473 )
14474 .await
14475 {
14476 Ok(_) => panic!("setup must fail"),
14477 Err(e) => e,
14478 };
14479 assert_eq!(
14480 error
14481 .downcast_ref::<crate::run::StartupFailure>()
14482 .unwrap()
14483 .run,
14484 id
14485 );
14486 let failed = RunState::load(&id).unwrap();
14487 assert_eq!(failed.status, RunStatus::Failed);
14488 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14489 assert!(Runner::resume(&id).is_err());
14490 runner.state.status = RunStatus::Ready;
14491 runner.state.driver_exited = true;
14492 runner.state.save().unwrap();
14493 let mut retry = runner_at(RunStatus::Reviewing);
14494 retry.state.repo = repo.clone();
14495 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14496 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14497 reselected.approved_at = jiff::Timestamp::now();
14498 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14499 retry.state.pr_head = Some(reselected.branch.clone());
14500 retry.state.pr_recovery = Some(reselected.clone());
14501 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14502 .await
14503 .unwrap();
14504 let restarted = Runner::open_selected(
14505 &repo,
14506 retry.state,
14507 retry.roles,
14508 1,
14509 runner.state.base_commit.clone(),
14510 reselected,
14511 )
14512 .await
14513 .unwrap();
14514 assert_eq!(
14515 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14516 .await
14517 .unwrap(),
14518 head
14519 );
14520 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14521 assert_eq!(
14522 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14523 head
14524 );
14525 }
14526
14527 #[tokio::test]
14528 #[cfg(unix)]
14529 async fn fallback_and_interrupted_restoration_require_coordination() {
14530 use std::os::fd::AsRawFd as _;
14531 ask_test_home();
14532 let tmp = tempfile::tempdir().unwrap();
14533 let mut runner = registered_runner(tmp.path()).await;
14534 runner.state.config.graph.timeout_resource_wait = 1;
14535 let winner = runner.state.candidates[0].clone();
14536 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14537 let leftover = winner.worktree.join("leftover");
14538 std::fs::write(&leftover, "untrusted edits").unwrap();
14539 let next = tmp.path().join("next-fixer");
14540 let mut failed = spec("first");
14541 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14542 let mut successor = spec("second");
14543 successor.command = vec![
14544 "sh".into(),
14545 "-c".into(),
14546 format!("touch '{}'; echo done", next.display()),
14547 ];
14548 let attempts = vec![
14549 (failed.clone(), "first".into()),
14550 (successor.clone(), "second".into()),
14551 ];
14552 let lock_path = runner.state.repo.join(".git/updater.lock");
14553 let external = std::fs::File::create(&lock_path).unwrap();
14554 assert_eq!(
14555 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14556 0
14557 );
14558 let artifacts = tmp.path().join("artifacts");
14559 let build = |spec, seat| SeatJob {
14560 spec,
14561 seat,
14562 cwd: winner.worktree.clone(),
14563 prompt: "fixture".into(),
14564 timeout: Duration::from_secs(10),
14565 allow_write: true,
14566 sessions: false,
14567 artifacts: artifacts.clone(),
14568 stem: "fix".into(),
14569 handover: None,
14570 };
14571 let (_, _, out) = runner
14572 .ask_fixer_among(
14573 attempts.clone(),
14574 "graft",
14575 None,
14576 Some((&winner.worktree, &head)),
14577 build,
14578 )
14579 .await;
14580 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14581 assert!(leftover.exists(), "contention must prevent clean");
14582 assert!(!next.exists());
14583 assert!(runner.state.handovers.is_empty());
14584 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14586 from_run: "earlier".into(),
14587 generation: 1,
14588 mode: crate::recovery::Mode::VerifyRepair,
14589 reasons: Vec::new(),
14590 reviewed_head: None,
14591 rounds: Vec::new(),
14592 open_findings: Vec::new(),
14593 evidence: None,
14594 criteria_inherited: true,
14595 repair: crate::recovery::Repair::Started {
14596 before: head.clone(),
14597 },
14598 });
14599 assert!(
14600 runner
14601 .recovery_repair(&winner)
14602 .await
14603 .unwrap_err()
14604 .is::<crate::repo_lock::ResourceBlocked>()
14605 );
14606 assert!(matches!(
14607 runner.state.recovery.as_ref().unwrap().repair,
14608 crate::recovery::Repair::Started { .. }
14609 ));
14610 assert!(leftover.exists());
14611 drop(external);
14612 runner.recovery_repair(&winner).await.unwrap();
14613 assert!(!leftover.exists());
14614 assert!(matches!(
14615 runner.state.recovery.as_ref().unwrap().repair,
14616 crate::recovery::Repair::Done { .. }
14617 ));
14618 std::fs::remove_file(&lock_path).unwrap();
14620 std::fs::create_dir(&lock_path).unwrap();
14621 std::fs::write(&leftover, "untrusted edits").unwrap();
14622 let (_, _, out) = runner
14623 .ask_fixer_among(
14624 attempts,
14625 "graft",
14626 None,
14627 Some((&winner.worktree, &head)),
14628 build,
14629 )
14630 .await;
14631 assert!(matches!(out, AgentOutcome::Failed(_)));
14632 assert!(leftover.exists());
14633 assert!(!next.exists());
14634 assert_eq!(
14635 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14636 head
14637 );
14638 }
14639
14640 #[cfg(unix)]
14641 async fn registered_runner(root: &Path) -> Runner {
14642 repository_runner(root, crate::config::test_coordination()).await
14643 }
14644
14645 async fn repository_runner(
14646 root: &Path,
14647 coordination: Option<crate::config::Coordination>,
14648 ) -> Runner {
14649 ask_test_home();
14650 let repo = root.join("repo");
14651 std::fs::create_dir_all(&repo).unwrap();
14652 init_repo(&repo);
14653 if coordination.is_some() {
14654 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14655 }
14656 let mut runner = runner_at(RunStatus::Prep);
14657 runner.state.repo = repo.clone();
14658 runner.state.config.coordination = coordination;
14659 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14660 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14661 runner.state.config.graph.advise = false;
14662 runner.roles.implementers = vec![spec("implement")];
14663 runner.roles.judges = vec![spec("judge")];
14664 runner.prep().await.unwrap();
14665 runner
14666 }
14667
14668 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14669
14670 #[tokio::test]
14671 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14672 let tmp = tempfile::tempdir().unwrap();
14673 let mut runner = repository_runner(tmp.path(), None).await;
14674 let prompts = runner.state.config.prompts.clone();
14675 let run = runner.state.id.clone();
14676 let ctx = WaveCtx {
14677 carry_seats: false,
14678 pauses: None,
14679 run: &run,
14680 node: "advise",
14681 prompts: &prompts,
14682 cache: None,
14683 round: None,
14684 };
14685 let jobs: Vec<_> = ["alpha", "beta"]
14686 .into_iter()
14687 .map(|id| SeatJob {
14688 spec: scripted(id, AUTH_OUTAGE),
14689 seat: SeatState::new(id, id, 7),
14690 prompt: "propose".into(),
14691 cwd: runner.state.candidates[0].worktree.clone(),
14692 timeout: Duration::from_secs(10),
14693 allow_write: false,
14694 sessions: false,
14695 artifacts: agent::artifacts_dir(&runner.state.dir()),
14696 stem: id.into(),
14697 handover: None,
14698 })
14699 .collect();
14700 let mut losses = Vec::new();
14701 let results = ask_wave_with(
14702 jobs.clone(),
14703 runner.sem.clone(),
14704 2,
14705 &[],
14706 &ctx,
14707 &mut losses,
14708 &mut runner.state,
14709 &|_: &str| Ok(()),
14710 )
14711 .await;
14712 assert!(
14713 results
14714 .iter()
14715 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14716 );
14717 assert_eq!(runner.state.quota.len(), 2);
14718 assert!(
14719 runner
14720 .state
14721 .quota
14722 .iter()
14723 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14724 );
14725 assert!(!runner.state.advise_attempted);
14726 runner.stop_for_resource().unwrap();
14727 assert_eq!(runner.state.status, RunStatus::Blocked);
14728 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14729 assert_eq!(
14730 runner.state.resource_block.as_ref().unwrap().kind,
14731 "agent authentication"
14732 );
14733 runner.state.resource_block = None;
14736 runner.state.quota.clear();
14737 let mut mixed = jobs;
14738 mixed[1].spec.command = vec!["false".into()];
14739 let results = ask_wave_with(
14740 mixed,
14741 runner.sem.clone(),
14742 0,
14743 &[],
14744 &ctx,
14745 &mut losses,
14746 &mut runner.state,
14747 &|_: &str| Ok(()),
14748 )
14749 .await;
14750 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14751 assert!(runner.state.resource_block.is_none());
14752 assert_eq!(losses.len(), 1);
14753 }
14754
14755 #[tokio::test]
14756 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14757 let tmp = tempfile::tempdir().unwrap();
14758 let mut runner = repository_runner(tmp.path(), None).await;
14759 let implementer = scripted("implement", AUTH_OUTAGE);
14760 runner.roles.implementers = vec![implementer.clone()];
14761 runner.state.config.agents = vec![implementer];
14762 runner.implement().await.unwrap();
14763 assert_eq!(runner.state.status, RunStatus::Blocked);
14764 assert_eq!(runner.state.quota.len(), 1);
14765 assert_eq!(
14766 runner.state.quota[0].cause,
14767 agent::QuotaCause::Authentication
14768 );
14769 let candidate = &runner.state.candidates[0];
14770 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14771 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14772 }
14773
14774 #[tokio::test]
14775 #[cfg(unix)]
14776 async fn parked_implementer_persists_completed_work() {
14777 for registered in [false, true] {
14778 let tmp = tempfile::tempdir().unwrap();
14779 let mut runner = registered_runner(tmp.path()).await;
14780 if !registered {
14781 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14782 }
14783 let marker = tmp.path().join("started");
14784 let release = tmp.path().join("release");
14785 let mut implementer = spec("implement");
14786 implementer.command = vec![
14787 "sh".into(),
14788 "-c".into(),
14789 format!(
14790 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14791 marker.display(),
14792 release.display()
14793 ),
14794 ];
14795 runner.roles.implementers = vec![implementer.clone()];
14796 runner.state.config.agents = vec![implementer];
14797 let pause = runner.pause.clone();
14798 let (result, ()) = tokio::join!(runner.implement(), async {
14799 tokio::time::timeout(Duration::from_secs(10), async {
14800 while !marker.exists() {
14801 tokio::time::sleep(Duration::from_millis(10)).await;
14802 }
14803 })
14804 .await
14805 .unwrap();
14806 pause.park();
14807 std::fs::write(&release, "go").unwrap();
14808 });
14809 result.unwrap();
14810 let candidate = &runner.state.candidates[0];
14811 assert!(candidate.commits > 0, "{candidate:?}");
14812 assert!(candidate.summary.contains("saved-result"));
14813 assert_eq!(candidate.files, 1);
14814 let before = candidate.commits;
14815 runner.implement().await.unwrap();
14816 assert_eq!(runner.state.candidates[0].commits, before);
14817 }
14818 }
14819
14820 #[tokio::test]
14821 #[cfg(unix)]
14822 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14823 let tmp = tempfile::tempdir().unwrap();
14824 let mut runner = registered_runner(tmp.path()).await;
14825 let winner = runner.state.candidates[0].clone();
14826 let cache = tmp.path().join("cache");
14827 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14828 runner.state.config.graph.timeout_resource_wait = 0;
14829 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14830 let crate::cache::AcquireOutcome::Acquired(held) =
14831 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14832 else {
14833 panic!("cache must be free")
14834 };
14835 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14836 from_run: "earlier".into(),
14837 generation: 1,
14838 mode: crate::recovery::Mode::VerifyRepair,
14839 reasons: Vec::new(),
14840 reviewed_head: None,
14841 rounds: Vec::new(),
14842 open_findings: Vec::new(),
14843 evidence: None,
14844 criteria_inherited: true,
14845 repair: crate::recovery::Repair::NotStarted,
14846 });
14847 runner.recovery_repair(&winner).await.unwrap();
14848 assert!(matches!(
14849 runner.state.recovery.as_ref().unwrap().repair,
14850 crate::recovery::Repair::WaitingForRestore { .. }
14851 ));
14852 drop(held);
14853 runner.state.resource_block = None;
14854 runner.state.config.graph.timeout_resource_wait = 1;
14855 runner.recovery_repair(&winner).await.unwrap();
14856 assert!(matches!(
14857 runner.state.recovery.as_ref().unwrap().repair,
14858 crate::recovery::Repair::Done { .. }
14859 ));
14860 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14861 }
14862
14863 #[tokio::test]
14864 #[cfg(unix)]
14865 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14866 use std::os::fd::AsRawFd as _;
14867 for graft in [false, true] {
14868 let tmp = tempfile::tempdir().unwrap();
14869 let mut runner = registered_runner(tmp.path()).await;
14870 runner.state.config.graph.timeout_resource_wait = 1;
14871 let winner = runner.state.candidates[0].clone();
14872 let marker = tmp.path().join("fixer-started");
14873 let mut fixer = spec("fixer");
14874 fixer.command = vec![
14875 "sh".into(),
14876 "-c".into(),
14877 format!("echo started >> '{}'; echo done", marker.display()),
14878 ];
14879 runner.roles.fixer = Some(vec![fixer]);
14880 if graft {
14881 let mut loser = winner.clone();
14882 loser.label = 'B';
14883 loser.index = 1;
14884 runner.state.candidates.push(loser);
14885 runner.state.config.graph.graft_runner_up = true;
14886 runner.state.tally = Some(Tally {
14887 first_choice: BTreeMap::new(),
14888 borda: BTreeMap::new(),
14889 winner: winner.label,
14890 rankings: 1,
14891 unanimous_initial: true,
14892 deliberated: false,
14893 changed_votes: 0,
14894 unanimous_final: true,
14895 tie_break: None,
14896 judges: 1,
14897 present: 1,
14898 quorum: 1,
14899 met_quorum: true,
14900 uncontested: None,
14901 });
14902 runner.state.judgements.push(crate::run::Judgement {
14903 judge: 1,
14904 seat: "judge-1".into(),
14905 agent: "judge".into(),
14906 ranking: vec![winner.label, 'B'],
14907 reasons: BTreeMap::new(),
14908 confidence: None,
14909 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14910 order: vec![0, 1],
14911 failed: None,
14912 duration_ms: 0,
14913 });
14914 } else {
14915 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14916 from_run: "earlier".into(),
14917 generation: 1,
14918 mode: crate::recovery::Mode::VerifyRepair,
14919 reasons: Vec::new(),
14920 reviewed_head: None,
14921 rounds: Vec::new(),
14922 open_findings: Vec::new(),
14923 evidence: None,
14924 criteria_inherited: true,
14925 repair: crate::recovery::Repair::NotStarted,
14926 });
14927 }
14928 let reader = crate::repo_lock::SeatLease::for_checkout(
14929 &winner.worktree,
14930 false,
14931 crate::config::test_coordination().as_ref(),
14932 )
14933 .await
14934 .unwrap();
14935 let pause = runner.pause.clone();
14936 let (result, ()) = tokio::join!(
14937 async {
14938 if graft {
14939 runner.graft().await
14940 } else {
14941 runner.recovery_repair(&winner).await
14942 }
14943 },
14944 async {
14945 tokio::time::sleep(Duration::from_millis(100)).await;
14946 pause.park();
14947 }
14948 );
14949 result.unwrap();
14950 assert!(!marker.exists(), "a blocked seat must not spawn");
14951 runner.state = RunState::load(&runner.state.id).unwrap();
14953 if graft {
14954 assert_eq!(
14955 runner.state.graft.as_ref().unwrap().outcome,
14956 GraftOutcome::WaitingForRestore
14957 );
14958 } else {
14959 assert!(matches!(
14960 runner.state.recovery.as_ref().unwrap().repair,
14961 crate::recovery::Repair::WaitingForRestore { .. }
14962 ));
14963 }
14964 drop(reader);
14965 runner.pause = Pause::new();
14966 runner.state.resource_block = None;
14967 let external =
14969 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14970 assert_eq!(
14971 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14972 0
14973 );
14974 let leftover = winner.worktree.join("leftover");
14975 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14976 let result = if graft {
14977 runner.graft().await
14978 } else {
14979 runner.recovery_repair(&winner).await
14980 };
14981 assert!(
14982 result
14983 .unwrap_err()
14984 .is::<crate::repo_lock::ResourceBlocked>()
14985 );
14986 assert!(leftover.exists());
14987 assert!(!marker.exists());
14988 drop(external);
14989 runner.state.resource_block = None;
14990 let result = if graft {
14991 runner.graft().await
14992 } else {
14993 runner.recovery_repair(&winner).await
14994 };
14995 result.unwrap();
14996 assert!(!leftover.exists(), "restore must precede retry");
14997 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14998 if graft {
14999 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15000 } else {
15001 assert!(matches!(
15002 runner.state.recovery.as_ref().unwrap().repair,
15003 crate::recovery::Repair::Done { .. }
15004 ));
15005 }
15006 }
15007 }
15008
15009 #[tokio::test]
15010 #[cfg(unix)]
15011 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15012 use std::os::fd::AsRawFd as _;
15013 let tmp = tempfile::tempdir().unwrap();
15014 let mut runner = registered_runner(tmp.path()).await;
15015 runner.state.config.graph.timeout_resource_wait = 1;
15016 let winner = runner.state.candidates[0].clone();
15017 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15018 runner.state.tally = Some(Tally {
15019 first_choice: BTreeMap::new(),
15020 borda: BTreeMap::new(),
15021 winner: winner.label,
15022 rankings: 0,
15023 unanimous_initial: false,
15024 deliberated: false,
15025 changed_votes: 0,
15026 unanimous_final: false,
15027 tie_break: None,
15028 judges: 0,
15029 present: 0,
15030 quorum: 0,
15031 met_quorum: true,
15032 uncontested: Some("fixture".into()),
15033 });
15034 runner.state.graft = Some(GraftRecord {
15035 pre: pre.clone(),
15036 post: None,
15037 lines: None,
15038 limit: None,
15039 agent: None,
15040 outcome: GraftOutcome::Running,
15041 requested: Vec::new(),
15042 adoption: None,
15043 adoption_error: None,
15044 });
15045 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15046 git::commit_all(&winner.worktree, "unfinished graft")
15047 .await
15048 .unwrap();
15049 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15050 let leftover = winner.worktree.join("leftover");
15051 std::fs::write(&leftover, "untrusted edits").unwrap();
15052 let path = runner.state.repo.join(".git/updater.lock");
15053 let external = std::fs::File::create(&path).unwrap();
15054 assert_eq!(
15055 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15056 0
15057 );
15058 assert!(
15059 runner
15060 .graft()
15061 .await
15062 .unwrap_err()
15063 .is::<crate::repo_lock::ResourceBlocked>()
15064 );
15065 assert!(matches!(
15066 runner.state.graft.as_ref().unwrap().outcome,
15067 GraftOutcome::Running
15068 ));
15069 assert_eq!(
15070 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15071 unfinished
15072 );
15073 assert!(leftover.exists());
15074 drop(external);
15075 std::fs::remove_file(&path).unwrap();
15076 std::fs::create_dir(&path).unwrap();
15077 assert!(runner.graft().await.is_err());
15078 assert!(matches!(
15079 runner.state.graft.as_ref().unwrap().outcome,
15080 GraftOutcome::Running
15081 ));
15082 assert_eq!(
15083 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15084 unfinished
15085 );
15086 assert!(leftover.exists());
15087 std::fs::remove_dir(&path).unwrap();
15088 runner.graft().await.unwrap();
15089 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15090 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15091 assert!(!leftover.exists());
15092 }
15093
15094 #[tokio::test]
15095 #[cfg(unix)]
15096 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15097 use std::os::fd::AsRawFd as _;
15098 let tmp = tempfile::tempdir().unwrap();
15099 let mut runner = registered_runner(tmp.path()).await;
15100 runner.state.prep_complete = Some(false);
15101 runner.state.config.graph.timeout_resource_wait = 1;
15102 runner.state.save().unwrap();
15103 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15104 assert_eq!(
15105 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15106 0
15107 );
15108 runner.execute().await.unwrap();
15109 let disk = RunState::load(&runner.state.id).unwrap();
15110 assert!(disk.driver_exited);
15111 assert_eq!(disk.driver_pid, None);
15112 assert_eq!(runner.state.status, RunStatus::Blocked);
15113 assert_eq!(
15114 runner.state.resource_block.as_ref().unwrap().kind,
15115 "repository coordination"
15116 );
15117 assert!(runner.state.prep_failure.is_none());
15118 assert!(runner.state.seats.is_empty());
15119 runner.pause.park();
15120 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15123 .await
15124 .unwrap()
15125 .unwrap();
15126 assert!(runner.state.parked);
15127 assert!(runner.state.prep_failure.is_none());
15128 let disk = RunState::load(&runner.state.id).unwrap();
15129 assert!(disk.driver_exited);
15130 assert_eq!(disk.driver_pid, None);
15131 }
15132
15133 #[tokio::test]
15134 #[cfg(unix)]
15135 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15136 use std::os::fd::AsRawFd as _;
15137 let tmp = tempfile::tempdir().unwrap();
15138 let mut runner = registered_runner(tmp.path()).await;
15139 runner.state.prep_complete = Some(false);
15140 runner.state.config.graph.timeout_resource_wait = 30;
15141 runner.state.save().unwrap();
15142 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15143 assert_eq!(
15144 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15145 0
15146 );
15147 assert!(
15148 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15149 .await
15150 .is_err()
15151 );
15152 let disk = RunState::load(&runner.state.id).unwrap();
15153 assert!(disk.driver_exited);
15154 assert_eq!(disk.driver_pid, None);
15155 assert!(disk.status.resumable());
15156 assert!(runner.state.driver_exited);
15157 assert_eq!(runner.state.driver_pid, None);
15158 runner.state.save().unwrap();
15159 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15160 }
15161
15162 #[tokio::test]
15163 #[cfg(unix)]
15164 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15165 let tmp = tempfile::tempdir().unwrap();
15166 let mut runner = registered_runner(tmp.path()).await;
15167 let winner = runner.state.candidates[0].clone();
15168 let cache = tmp.path().join("cache");
15169 let started = tmp.path().join("pre-gate-started");
15170 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15171 runner.state.config.verify.gate =
15172 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15173 let home = crate::run::home();
15174 let owner =
15175 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15176 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15177 crate::cache::AcquireOutcome::Acquired(g) => g,
15178 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15179 };
15180 let mut gate = Box::pin(runner.run_gate(&winner));
15181 assert!(
15182 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15183 .await
15184 .is_err()
15185 );
15186 assert!(!started.exists(), "pre-gate must wait for the cache too");
15187 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15191 let editor = crate::repo_lock::SeatLease::checkout_with(
15192 &winner.worktree,
15193 true,
15194 crate::config::test_coordination().as_ref(),
15195 &wait,
15196 )
15197 .await
15198 .unwrap();
15199 drop(editor);
15200 guard.release();
15201 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15202 .await
15203 .unwrap()
15204 .unwrap();
15205 drop(gate);
15206 assert!(outcomes.iter().all(CommandOutcome::ok));
15207 assert!(started.exists());
15208 }
15209
15210 #[tokio::test]
15211 #[cfg(unix)]
15212 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15213 use std::os::fd::AsRawFd as _;
15214
15215 ask_test_home();
15216 let tmp = tempfile::tempdir().unwrap();
15217 let repo = tmp.path().join("repo");
15218 std::fs::create_dir_all(&repo).unwrap();
15219 init_repo(&repo);
15220 let common = repo.join(".git");
15221 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15222 let mut runner = runner_at(RunStatus::Prep);
15223 runner.state.repo = repo.clone();
15224 runner.state.config.coordination = crate::config::test_coordination();
15225 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15226 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15227 runner.state.config.graph.advise = false;
15228 runner.roles.implementers = vec![spec("implement")];
15229 runner.roles.judges = vec![spec("judge")];
15230 runner.prep().await.unwrap();
15231 let winner = runner.state.candidates[0].clone();
15232 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15235 let completed = tmp.path().join("seat-completed");
15236 runner.roles.implementers[0].command = vec![
15237 "sh".into(),
15238 "-c".into(),
15239 format!("touch '{}'; echo completed", completed.display()),
15240 ];
15241 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15242 assert_eq!(
15243 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15244 0
15245 );
15246 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15247 let mut implement = Box::pin(runner.implement());
15248 tokio::select! {
15249 result = &mut implement => panic!("rescue must wait: {result:?}"),
15250 result = tokio::time::timeout(Duration::from_secs(10), async {
15251 while !completed.exists() {
15252 tokio::time::sleep(Duration::from_millis(20)).await;
15253 }
15254 }) => result.unwrap(),
15255 }
15256 assert!(
15257 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15258 .await
15259 .is_err()
15260 );
15261 assert_eq!(
15262 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15263 before
15264 );
15265 assert_eq!(
15266 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15267 0
15268 );
15269 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15270 .await
15271 .unwrap()
15272 .unwrap();
15273 drop(implement);
15274 assert_ne!(
15275 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15276 before
15277 );
15278 assert!(git::is_clean(&winner.worktree).await.unwrap());
15279
15280 let started = tmp.path().join("pre-gate-started");
15281 let proceed = tmp.path().join("pre-gate-proceed");
15282 let checked = tmp.path().join("gate-ran");
15283 runner.state.config.verify.pre_gate = vec![format!(
15284 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15285 started.display(),
15286 proceed.display()
15287 )];
15288 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15289 assert_eq!(
15290 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15291 0
15292 );
15293 let mut gate = Box::pin(runner.run_gate(&winner));
15294 assert!(
15295 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15296 .await
15297 .is_err()
15298 );
15299 assert!(!started.exists());
15300 assert!(!checked.exists());
15301 assert!(git::is_clean(&winner.worktree).await.unwrap());
15302 assert_eq!(
15303 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15304 0
15305 );
15306 tokio::select! {
15307 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15308 result = tokio::time::timeout(Duration::from_secs(10), async {
15309 while !started.exists() {
15310 tokio::time::sleep(Duration::from_millis(20)).await;
15311 }
15312 }) => result.unwrap(),
15313 }
15314 assert_eq!(
15315 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15316 -1,
15317 "mechanical edits must hold the real lock"
15318 );
15319 std::fs::write(proceed, "ready").unwrap();
15320 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15321 .await
15322 .unwrap()
15323 .unwrap();
15324 drop(gate);
15325 assert!(outcomes.iter().all(CommandOutcome::ok));
15326 assert!(checked.exists());
15327 assert!(runner.state.pre_gate_commit.is_some());
15328 assert!(git::is_clean(&winner.worktree).await.unwrap());
15329 assert_eq!(
15330 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15331 0,
15332 "gate completion must release coordination"
15333 );
15334 drop(outsider);
15335 std::fs::remove_file(common.join("updater.lock")).unwrap();
15336 std::fs::create_dir(common.join("updater.lock")).unwrap();
15337 std::fs::remove_file(&started).unwrap();
15338 std::fs::remove_file(&checked).unwrap();
15339 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15340 let error = runner.run_gate(&winner).await.unwrap_err();
15341 assert!(
15342 error
15343 .to_string()
15344 .contains("acquire mandatory repository lock")
15345 );
15346 assert!(
15347 !started.exists(),
15348 "failed acquisition must prevent pre-gate"
15349 );
15350 assert!(!checked.exists(), "failed acquisition must prevent gate");
15351 assert!(git::is_clean(&winner.worktree).await.unwrap());
15352 assert_eq!(
15353 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15354 head
15355 );
15356 }
15357
15358 #[tokio::test]
15359 #[cfg(unix)]
15360 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15361 use std::os::fd::AsRawFd as _;
15362 use std::time::Duration;
15363
15364 ask_test_home();
15365 let tmp = tempfile::tempdir().unwrap();
15366 let repo = tmp.path().join("repo");
15367 std::fs::create_dir_all(&repo).unwrap();
15368 init_repo(&repo);
15369 let common = repo.join(".git");
15370 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15371 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15372 assert_eq!(
15373 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15374 0
15375 );
15376 let mut runner = runner_at(RunStatus::Prep);
15377 runner.state.repo = repo.clone();
15378 runner.state.config.coordination = crate::config::test_coordination();
15379 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15380 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15381 runner.state.config.graph.advise = true;
15382 runner.state.config.graph.advisors = 1;
15383 runner.roles.implementers = vec![spec("implement")];
15384 runner.roles.judges = vec![spec("judge")];
15385 let root = runner.state.worktree_root();
15386 let started = tmp.path().join("setup-started");
15387 let proceed = tmp.path().join("setup-proceed");
15388 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15389 run: Some(format!(
15390 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15391 started.display(),
15392 proceed.display()
15393 )),
15394 ..Default::default()
15395 }];
15396 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15397 let config = git::git(&repo, &["config", "--local", "--list"])
15398 .await
15399 .unwrap();
15400 let mut prep = Box::pin(runner.prep());
15401 assert!(
15402 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15403 .await
15404 .is_err()
15405 );
15406 assert!(
15407 !root.exists(),
15408 "contention must prevent even checkout creation"
15409 );
15410 assert!(!started.exists(), "contention must prevent setup");
15411 assert_eq!(
15412 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15413 branches
15414 );
15415 assert_eq!(
15416 git::git(&repo, &["config", "--local", "--list"])
15417 .await
15418 .unwrap(),
15419 config
15420 );
15421 drop(lock);
15422 tokio::select! {
15423 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15424 result = tokio::time::timeout(Duration::from_secs(10), async {
15425 while !started.exists() {
15426 tokio::time::sleep(Duration::from_millis(20)).await;
15427 }
15428 }) => result.unwrap(),
15429 }
15430 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15431 assert_eq!(
15432 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15433 -1
15434 );
15435 assert!(
15436 tokio::time::timeout(
15437 Duration::from_millis(100),
15438 crate::repo_lock::SeatLease::for_checkout(
15439 &repo,
15440 true,
15441 crate::config::test_coordination().as_ref()
15442 )
15443 )
15444 .await
15445 .is_err(),
15446 "preparation must also exclude local editors"
15447 );
15448 std::fs::write(proceed, "ready").unwrap();
15449 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15450 .await
15451 .unwrap()
15452 .unwrap();
15453 drop(prep);
15454 assert_eq!(runner.state.prep_complete, Some(true));
15455 assert!(root.join("judge-1").exists());
15456 assert!(root.join("advisor-1").exists());
15457 assert_eq!(
15458 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15459 0
15460 );
15461 }
15462
15463 #[tokio::test]
15464 #[cfg(unix)]
15465 async fn preparation_lock_failure_prevents_repository_changes() {
15466 ask_test_home();
15467 let tmp = tempfile::tempdir().unwrap();
15468 let repo = tmp.path().join("repo");
15469 std::fs::create_dir_all(&repo).unwrap();
15470 init_repo(&repo);
15471 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15472 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15475 let mut runner = runner_at(RunStatus::Prep);
15476 runner.state.repo = repo.clone();
15477 runner.state.config.coordination = crate::config::test_coordination();
15478 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15479 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15480 runner.roles.implementers = vec![spec("implement")];
15481 let error = runner.prep().await.unwrap_err();
15482 assert!(
15483 error
15484 .to_string()
15485 .contains("acquire mandatory repository lock")
15486 );
15487 assert!(!runner.state.worktree_root().exists());
15488 assert!(runner.state.candidates.is_empty());
15489 assert!(!repo.join(".git/config.worktree").exists());
15490 assert_eq!(
15491 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15492 "* main"
15493 );
15494 }
15495
15496 #[tokio::test]
15497 async fn preparation_records_creation_intent_before_git_can_fail() {
15498 ask_test_home();
15499 let tmp = tempfile::tempdir().unwrap();
15500 let repo = tmp.path().join("repo");
15501 std::fs::create_dir_all(&repo).unwrap();
15502 init_repo(&repo);
15503 let mut runner = runner_at(RunStatus::Prep);
15504 runner.state.repo = repo.clone();
15505 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15506 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15507 runner.state.config.blind.commit_msg_hook = false;
15508 runner.roles.implementers = vec![spec("one")];
15509 let label = blind::assign_labels(1, runner.state.seed)[0];
15510 let branch = runner.state.branch_for(label);
15511 git::git(&repo, &["branch", &branch]).await.unwrap();
15514 assert!(runner.prep().await.is_err());
15515 let saved = RunState::load(&runner.state.id).unwrap();
15516 assert!(saved.candidates.is_empty());
15517 let event = saved
15518 .events
15519 .iter()
15520 .find(|e| e.node == "prep-intent")
15521 .unwrap();
15522 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15523 assert_eq!(intent["branch"], branch);
15524 assert_eq!(intent["start"], runner.state.base_commit);
15525 assert_eq!(
15526 intent["worktree"],
15527 runner
15528 .state
15529 .worktree_root()
15530 .join(format!("cand-{label}"))
15531 .to_string_lossy()
15532 .as_ref()
15533 );
15534 assert_eq!(intent["picks"], serde_json::json!([]));
15535 }
15536
15537 #[tokio::test]
15538 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15539 ask_test_home();
15540 let tmp = tempfile::tempdir().unwrap();
15541 let repo = tmp.path().join("repo");
15542 std::fs::create_dir_all(&repo).unwrap();
15543 init_repo(&repo);
15544 let mut runner = runner_at(RunStatus::Prep);
15545 runner.state.repo = repo.clone();
15546 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15547 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15548 runner.state.sources.pick_commits.push(refs::Input {
15549 token: "missing".into(),
15550 sha: "0".repeat(40),
15551 provenance: refs::Provenance::ExplicitCli,
15552 });
15553 assert!(runner.prep().await.is_err());
15554 assert!(runner.state.candidates.is_empty());
15555 assert!(!runner.state.worktree_root().exists());
15556 }
15557
15558 #[tokio::test]
15559 async fn prose_and_legacy_inference_never_import_into_candidates() {
15560 ask_test_home();
15561 let tmp = tempfile::tempdir().unwrap();
15562 let repo = tmp.path().join("repo");
15563 std::fs::create_dir_all(&repo).unwrap();
15564 init_repo(&repo);
15565 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15566 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15567 .await
15568 .unwrap();
15569 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15570 git::git(&repo, &["add", "."]).await.unwrap();
15571 git::git(&repo, &["commit", "-m", "historical implementation"])
15572 .await
15573 .unwrap();
15574 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15575 git::git(&repo, &["checkout", "main"]).await.unwrap();
15576 for instruction in [
15577 format!("Example: magi/history/A at {source}"),
15578 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15579 format!("Never import magi/history/A or {source}"),
15580 format!("Conversation answer: I mentioned {source} only as background"),
15581 ] {
15582 let mut runner = runner_at(RunStatus::Prep);
15583 runner.state.repo = repo.clone();
15584 runner.state.base_commit = base.clone();
15585 runner.state.instruction = instruction.clone();
15586 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15587 runner.state.config.blind.commit_msg_hook = false;
15588 runner.state.seeds = vec![refs::Seed {
15589 token: "magi/history/A".into(),
15590 sha: source.clone(),
15591 kind: refs::SeedKind::Unmerged,
15592 branch: true,
15593 detail: String::new(),
15594 }];
15595 runner.roles.implementers = vec![spec("one")];
15596 assert_eq!(seeded_instruction(&runner.state), instruction);
15597 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15599 legacy.as_object_mut().unwrap().remove("sources");
15600 runner.state = serde_json::from_value(legacy).unwrap();
15601 assert_eq!(runner.state.sources, refs::Sources::default());
15602 runner.state.save().unwrap();
15603 runner.state = RunState::load(&runner.state.id).unwrap();
15604 runner.prep().await.unwrap();
15605 for candidate in &runner.state.candidates {
15606 assert_eq!(
15607 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15608 base
15609 );
15610 assert!(!candidate.worktree.join("unwanted").exists());
15611 }
15612 }
15613 }
15614
15615 #[tokio::test]
15616 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15617 ask_test_home();
15618 let tmp = tempfile::tempdir().unwrap();
15619 let repo = tmp.path().join("repo");
15620 std::fs::create_dir_all(&repo).unwrap();
15621 init_repo(&repo);
15622 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15623 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15624 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15625 git::git(&repo, &["add", "."]).await.unwrap();
15626 git::git(&repo, &["commit", "-m", "required seed"])
15627 .await
15628 .unwrap();
15629 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15630 git::git(&repo, &["checkout", "main"]).await.unwrap();
15631 let mut runner = runner_at(RunStatus::Prep);
15632 runner.state.repo = repo.clone();
15633 runner.state.base_commit = base.clone();
15634 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15635 runner.state.config.graph.advise = false;
15636 runner.state.config.blind.commit_msg_hook = false;
15637 runner.state.sources =
15638 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15639 .await
15640 .unwrap();
15641 runner.roles.implementers = vec![spec("one"), spec("two")];
15642 runner.roles.judges = vec![spec("judge")];
15643 let label = blind::assign_labels(2, runner.state.seed)[0];
15644 let branch = runner.state.branch_for(label);
15645 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15646 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15647 .await
15648 .unwrap();
15649 runner.state.candidates.push(Candidate {
15650 index: 0,
15651 label,
15652 agent: "one".into(),
15653 branch,
15654 worktree,
15655 summary: String::new(),
15656 stat: String::new(),
15657 files: 0,
15658 commits: 0,
15659 empty: false,
15660 failed: None,
15661 verified_noop: None,
15662 duration_ms: 0,
15663 folded: false,
15664 });
15665 runner.state.save().unwrap();
15666 runner.state = RunState::load(&runner.state.id).unwrap();
15668 assert_eq!(runner.state.prep_complete, Some(false));
15669 runner.prep().await.unwrap();
15670 assert_eq!(runner.state.prep_complete, Some(true));
15671 assert_eq!(runner.state.candidates.len(), 2);
15672 for candidate in &runner.state.candidates {
15673 assert_eq!(
15674 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15675 git::tree_of(&repo, &seed).await.unwrap()
15676 );
15677 assert_eq!(
15678 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15679 "required commit"
15680 );
15681 }
15682 assert!(runner.state.worktree_root().join("judge-1").exists());
15683 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15684 runner.prep().await.unwrap();
15685 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15686
15687 runner.state.prep_complete = Some(false);
15688 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15689 run: Some(format!(
15690 "test -f '{}'",
15691 tmp.path().join("dependency-ready").display()
15692 )),
15693 ..Default::default()
15694 }];
15695 assert!(runner.prep().await.is_err());
15696 assert!(runner.state.prep_failure.is_some());
15697 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15698 runner.state = RunState::load(&runner.state.id).unwrap();
15699 assert!(
15700 runner.prep().await.is_err(),
15701 "correcting the cause alone must not auto-retry"
15702 );
15703 let failed_at = runner.state.events.last().unwrap().at;
15704 assert!(
15705 runner.retry_preparation(failed_at).is_err(),
15706 "old approvals cannot unlock the hold"
15707 );
15708 let mut task = crate::queue::Task::new(
15709 "retry".into(),
15710 "retry setup".into(),
15711 repo.clone(),
15712 crate::queue::Source::Human,
15713 );
15714 task.start(runner.state.id.clone());
15715 task.hold_machine(Some("setup failed".into()));
15716 task.release_operator("cli", None);
15717 runner
15718 .retry_preparation(task.resumption.as_ref().unwrap().at)
15719 .unwrap();
15720 runner.prep().await.unwrap();
15721 assert!(runner.state.prep_failure.is_none());
15722 assert_eq!(runner.state.prep_complete, Some(true));
15723 assert_eq!(runner.state.candidates.len(), 2);
15724 assert!(runner.state.worktree_root().join("judge-1").exists());
15725 for candidate in &runner.state.candidates {
15726 assert_eq!(
15727 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15728 git::tree_of(&repo, &seed).await.unwrap()
15729 );
15730 }
15731 }
15732
15733 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15734 let mut heads = Vec::new();
15735 for candidate in candidates {
15736 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15737 }
15738 heads
15739 }
15740
15741 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15742 ask_test_home();
15743 let tmp = tempfile::tempdir().unwrap();
15744 let repo = tmp.path().join("repo");
15745 std::fs::create_dir(&repo).unwrap();
15746 init_repo(&repo);
15747 let mut runner = runner_at(RunStatus::Reviewing);
15748 runner.state.repo = repo.clone();
15749 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15750 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15751 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15752 let branch = format!("candidate-{label}");
15753 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15754 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15755 .await
15756 .unwrap();
15757 if label == 'B' {
15758 std::fs::write(worktree.join("change"), "winner").unwrap();
15759 git::rescue_commit(&worktree, "candidate change")
15760 .await
15761 .unwrap();
15762 }
15763 runner.state.candidates.push(Candidate {
15764 index,
15765 label,
15766 agent: "alpha".into(),
15767 branch,
15768 worktree,
15769 summary: "completed reply".into(),
15770 stat: String::new(),
15771 files: usize::from(label == 'B'),
15772 commits: usize::from(label == 'B'),
15773 empty: label == 'A',
15774 failed: None,
15775 verified_noop: None,
15776 duration_ms: 1,
15777 folded: false,
15778 });
15779 }
15780 runner.state.judge_skipped = true;
15781 runner.state.tally = Some(Tally {
15782 first_choice: BTreeMap::new(),
15783 borda: BTreeMap::new(),
15784 winner: 'B',
15785 rankings: 0,
15786 unanimous_initial: false,
15787 deliberated: false,
15788 changed_votes: 0,
15789 unanimous_final: false,
15790 tie_break: None,
15791 judges: 0,
15792 present: 0,
15793 quorum: 0,
15794 met_quorum: true,
15795 uncontested: Some("only B produced a change".into()),
15796 });
15797 runner.state.save().unwrap();
15798 (tmp, runner)
15799 }
15800
15801 #[tokio::test]
15802 #[cfg(unix)]
15803 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15804 for (solo, failed_checkpoint) in
15805 [(true, false), (false, false), (true, true), (false, true)]
15806 {
15807 let (_tmp, mut runner) = preservation_resume_fixture().await;
15808 if solo {
15809 runner.state.candidates.retain(|c| c.label == 'A');
15810 }
15811 runner.state.candidates[0].empty = false;
15812 runner.state.candidates[0].summary.clear();
15813 runner.state.tally = None;
15814 runner.state.judge_skipped = false;
15815 runner.state.status = RunStatus::Implementing;
15816 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15817 let mut implementer = spec("alpha");
15818 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15819 runner.roles.implementers = vec![implementer];
15820 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15821 if failed_checkpoint {
15822 std::fs::create_dir_all(&patch).unwrap();
15823 }
15824 runner.state.save().unwrap();
15825 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15826 drop(pressure);
15827 runner.state = RunState::load(&runner.state.id).unwrap();
15828 assert_eq!(runner.state.status, RunStatus::Blocked);
15829 assert!(runner.state.interrupted_implementations.contains(&'A'));
15830 assert_eq!(runner.state.candidates[0].commits, 0);
15831 assert!(!runner.state.candidates[0].empty);
15832 assert!(runner.state.candidates[0].failed.is_none());
15833 if failed_checkpoint {
15834 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15835 std::fs::remove_dir(patch).unwrap();
15836 }
15837 runner.state.resource_block = None;
15838 runner.state.config.disk.min_free_bytes = 0;
15839 runner.roles.implementers[0].command = vec![
15840 "sh".into(),
15841 "-c".into(),
15842 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15843 ];
15844 runner.implement().await.unwrap();
15845 assert!(runner.state.interrupted_implementations.is_empty());
15846 assert!(runner.state.candidate_preservation.is_empty());
15847 assert_eq!(runner.state.candidates[0].commits, 1);
15848 assert!(!runner.state.candidates[0].empty);
15849 assert!(
15850 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15851 .unwrap()
15852 .contains("retried empty interrupted seat")
15853 );
15854 if !solo {
15855 assert_eq!(runner.state.candidates[1].commits, 1);
15856 }
15857 }
15858 }
15859
15860 #[tokio::test]
15861 #[cfg(unix)]
15862 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15863 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15864 let (_tmp, mut runner) = preservation_resume_fixture().await;
15865 runner.state.candidates.retain(|c| c.label == 'A');
15866 runner.state.candidates[0].empty = false;
15867 runner.state.candidates[0].summary.clear();
15868 runner.state.tally = None;
15869 runner.state.judge_skipped = false;
15870 runner.state.status = RunStatus::Implementing;
15871 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15874 let wt = runner.state.candidates[0].worktree.clone();
15875 std::fs::write(
15876 wt.join("interrupted-source"),
15877 "retain implementation work\n",
15878 )
15879 .unwrap();
15880 if committed {
15881 git::rescue_commit(&wt, "implementation checkpoint")
15882 .await
15883 .unwrap();
15884 }
15885 let lock = git::git(
15886 &wt,
15887 &[
15888 "rev-parse",
15889 "--path-format=absolute",
15890 "--git-path",
15891 "index.lock",
15892 ],
15893 )
15894 .await
15895 .unwrap();
15896 if block_rescue {
15897 std::fs::write(&lock, "fixture-owned lock").unwrap();
15898 }
15899 let mut implementer = spec("alpha");
15900 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15901 runner.roles.implementers = vec![implementer];
15902 runner.state.save().unwrap();
15903 let result = runner.implement().await;
15904 drop(pressure);
15905 assert_eq!(result.is_err(), block_rescue);
15906 assert_eq!(
15907 runner.state.resource_block.as_ref().unwrap().kind,
15908 "disk pressure"
15909 );
15910 assert!(
15911 runner
15912 .state
15913 .events
15914 .iter()
15915 .any(|e| e.message.contains("agent execution interrupted"))
15916 );
15917 assert!(
15918 runner
15919 .state
15920 .dir()
15921 .join("artifacts/candidate-A-recovery/staged.patch")
15922 .exists()
15923 );
15924 if block_rescue {
15925 let saved = RunState::load(&runner.state.id).unwrap();
15926 assert!(saved.candidate_preservation.contains_key(&'A'));
15927 assert!(!saved.candidates[0].empty);
15928 assert_eq!(
15929 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15930 "retain implementation work\n"
15931 );
15932 std::fs::remove_file(lock).unwrap();
15933 runner.state = saved;
15934 } else {
15935 assert_eq!(runner.state.status, RunStatus::Blocked);
15936 assert_eq!(runner.state.candidates[0].commits, 1);
15937 }
15938 runner.roles.implementers.clear();
15941 runner.state.config.disk.min_free_bytes = 0;
15942 runner.state.resource_block = None;
15943 runner.implement().await.unwrap();
15944 assert!(runner.state.candidate_preservation.is_empty());
15945 assert_eq!(runner.state.candidates[0].commits, 1);
15946 assert!(!runner.state.candidates[0].empty);
15947 assert!(git::is_clean(&wt).await.unwrap());
15948 assert!(
15949 crate::run::read_artifact(&runner.state, "cand-A.patch")
15950 .unwrap()
15951 .contains("retain implementation work")
15952 );
15953 }
15954 }
15955
15956 #[tokio::test]
15957 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15958 let (_tmp, mut runner) = preservation_resume_fixture().await;
15959 runner.implement().await.unwrap();
15961 assert!(runner.state.candidate_preservation.is_empty());
15962 assert!(
15963 !runner
15964 .state
15965 .dir()
15966 .join("artifacts/candidate-A-recovery")
15967 .exists()
15968 );
15969 runner.fold_losers().await.unwrap();
15970 assert!(!runner.state.candidates[0].worktree.exists());
15971 for schema in [14, crate::run::SCHEMA] {
15972 runner.state.schema = schema;
15973 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15974 runner.state.save().unwrap();
15975 runner.state = RunState::load(&runner.state.id).unwrap();
15976 runner.implement().await.unwrap();
15977 assert!(runner.state.candidate_preservation.is_empty());
15978 assert!(runner.state.candidates[0].folded);
15979 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15980 }
15981 }
15982
15983 #[tokio::test]
15984 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15985 let (_tmp, mut runner) = preservation_resume_fixture().await;
15986 runner.state.candidates.retain(|c| c.label == 'A');
15987 runner.state.tally = None;
15988 runner.state.judge_skipped = false;
15989 let candidate = &mut runner.state.candidates[0];
15990 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15991 candidate.failed = Some("deployment validation unavailable".into());
15992 std::fs::write(
15993 candidate.worktree.join("repair"),
15994 "future launcher policy\n",
15995 )
15996 .unwrap();
15997 runner
15998 .state
15999 .candidate_preservation
16000 .insert('A', "awaiting persistence".into());
16001 runner.persist_candidates().await.unwrap();
16002 let candidate = &runner.state.candidates[0];
16003 assert!(!candidate.empty);
16004 assert_eq!(candidate.commits, 1);
16005 assert_eq!(candidate.files, 1);
16006 assert!(candidate.failed.is_none());
16007 assert!(candidate.summary.contains("EPERM"));
16008 assert!(
16009 crate::run::read_artifact(&runner.state, "cand-A.patch")
16010 .unwrap()
16011 .contains("future launcher policy")
16012 );
16013 assert!(runner.state.candidate_preservation.is_empty());
16014 }
16015
16016 #[tokio::test]
16017 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16018 let (_tmp, mut runner) = preservation_resume_fixture().await;
16019 let wt = runner.state.candidates[0].worktree.clone();
16020 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16021 runner.state.schema = 14;
16022 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16023 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16024 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16025 for _ in 0..2 {
16026 assert!(runner.implement().await.is_err());
16027 assert_eq!(runner.state.status, RunStatus::Blocked);
16028 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16029 assert!(
16030 runner.state.candidates[0].empty,
16031 "old verdict must remain unusable"
16032 );
16033 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16034 assert!(runner.state.judge_skipped);
16035 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16036 assert!(
16037 crate::run::read_artifact(&runner.state, "cand-A.patch")
16038 .unwrap()
16039 .contains("owned edits")
16040 );
16041 assert!(wt.exists());
16042 assert_eq!(
16043 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16044 .await
16045 .unwrap(),
16046 1
16047 );
16048 runner.state = RunState::load(&runner.state.id).unwrap();
16049 }
16050 }
16051
16052 #[tokio::test]
16053 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16054 let (_tmp, mut runner) = preservation_resume_fixture().await;
16055 let wt = runner.state.candidates[0].worktree.clone();
16056 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16057 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16058 let index_lock = PathBuf::from(
16059 git::git(
16060 &wt,
16061 &[
16062 "rev-parse",
16063 "--path-format=absolute",
16064 "--git-path",
16065 "index.lock",
16066 ],
16067 )
16068 .await
16069 .unwrap(),
16070 );
16071 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16072 runner.state.schema = 14;
16073 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16074 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16075 assert!(runner.implement().await.is_err());
16076 assert!(runner.after_implement().is_err());
16077 assert!(runner.fold_losers().await.is_err());
16078 assert_eq!(
16079 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16080 "failed seat edits"
16081 );
16082 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16083 runner.state = RunState::load(&runner.state.id).unwrap();
16084 std::fs::remove_file(index_lock).unwrap();
16085 assert!(runner.implement().await.is_err());
16087 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16088 assert!(runner.state.candidates[0].empty);
16089 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16090 assert!(
16091 crate::run::read_artifact(&runner.state, "cand-A.patch")
16092 .unwrap()
16093 .contains("failed seat edits")
16094 );
16095 assert!(wt.exists());
16096 }
16097
16098 #[tokio::test]
16099 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16100 let (_tmp, mut runner) = preservation_resume_fixture().await;
16101 let repo = runner.state.repo.clone();
16102 let common = git::git(
16103 &repo,
16104 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16105 )
16106 .await
16107 .unwrap();
16108 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16109 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16110 assert!(runner.fold_losers().await.is_err());
16111 assert!(!runner.state.candidates[0].worktree.exists());
16112 assert!(!runner.state.candidates[0].folded);
16113 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16114 runner.state = RunState::load(&runner.state.id).unwrap();
16115 std::fs::remove_file(branch_lock).unwrap();
16116 let judge = runner.state.worktree_root().join("judge-1");
16117 git::worktree_add_detached(&repo, &judge, "main")
16118 .await
16119 .unwrap();
16120 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16121 runner.roles.judges = vec![spec("judge")];
16122 assert!(runner.fold_losers().await.is_err());
16123 runner.state = RunState::load(&runner.state.id).unwrap();
16124 assert!(
16125 runner.state.candidates[0].folded,
16126 "candidate progress saved before judge failure"
16127 );
16128 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16129 assert!(judge.join("uncommitted").exists());
16130 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16131 runner.fold_losers().await.unwrap();
16132 assert!(!judge.exists());
16133 runner.fold_losers().await.unwrap();
16134 }
16135
16136 #[tokio::test]
16137 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16138 let (_tmp, mut runner) = preservation_resume_fixture().await;
16139 runner.state.candidates.retain(|c| c.label == 'B');
16140 runner.state.tally = None;
16141 runner.state.judge_skipped = false;
16142 runner.state.prep_complete = Some(true);
16143 runner.state.config.graph.advise = false;
16144 runner.state.config.graph.selected_implementer = Some("alpha".into());
16145 let wt = runner.state.candidates[0].worktree.clone();
16146 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16147 runner
16148 .state
16149 .candidate_preservation
16150 .insert('B', "awaiting persistence".into());
16151 let index_lock = git::git(
16152 &wt,
16153 &[
16154 "rev-parse",
16155 "--path-format=absolute",
16156 "--git-path",
16157 "index.lock",
16158 ],
16159 )
16160 .await
16161 .unwrap();
16162 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16163 runner.state.save().unwrap();
16164 assert!(runner.execute_graph().await.is_err());
16165 assert_eq!(runner.state.status, RunStatus::Blocked);
16166 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16167 std::fs::remove_file(index_lock).unwrap();
16168 for _ in 0..2 {
16171 runner.state = RunState::load(&runner.state.id).unwrap();
16172 runner.pause.park();
16173 runner.execute_graph().await.unwrap();
16174 assert_eq!(runner.state.status, RunStatus::Implementing);
16175 assert!(runner.state.candidate_preservation.is_empty());
16176 assert_eq!(runner.state.candidates[0].commits, 2);
16177 assert!(runner.state.tally.is_none());
16178 }
16179 runner.implement().await.unwrap();
16180 assert_eq!(runner.state.status, RunStatus::Judging);
16181 runner.state.status = RunStatus::Blocked;
16183 runner.state.candidates[0].failed = Some("implementation failed".into());
16184 runner.state.save().unwrap();
16185 let error = runner.execute_graph().await.unwrap_err();
16186 assert!(
16187 error
16188 .to_string()
16189 .contains("selected implementation is held")
16190 );
16191 }
16192
16193 #[tokio::test]
16194 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16195 let (_tmp, mut runner) = preservation_resume_fixture().await;
16196 let repo = runner.state.repo.clone();
16197 let candidate = runner.state.candidates[0].clone();
16198 let registered = git::worktree_holding(&repo, &candidate.branch)
16201 .await
16202 .unwrap()
16203 .unwrap();
16204 assert_eq!(
16205 registered.canonicalize().unwrap(),
16206 candidate.worktree.canonicalize().unwrap()
16207 );
16208 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16209 assert_eq!(
16210 git::worktree_holding(&repo, &candidate.branch)
16211 .await
16212 .unwrap(),
16213 Some(registered)
16214 );
16215 assert!(
16216 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16217 "the stale registration initially keeps the branch checked out"
16218 );
16219 runner.fold_losers().await.unwrap();
16220 assert!(runner.state.candidates[0].folded);
16221 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16222 assert_eq!(
16223 git::worktree_holding(&repo, &candidate.branch)
16224 .await
16225 .unwrap(),
16226 None
16227 );
16228 runner.state = RunState::load(&runner.state.id).unwrap();
16229 runner.fold_losers().await.unwrap();
16230 }
16231
16232 #[tokio::test]
16233 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16234 let (_tmp, mut runner) = preservation_resume_fixture().await;
16235 let repo = runner.state.repo.clone();
16236 let candidate = runner.state.candidates[0].clone();
16237 let winner = runner.state.candidates[1].clone();
16238 let registered = git::worktree_holding(&repo, &candidate.branch)
16239 .await
16240 .unwrap()
16241 .unwrap();
16242 assert_eq!(
16243 registered.canonicalize().unwrap(),
16244 candidate.worktree.canonicalize().unwrap()
16245 );
16246 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16247 assert_eq!(
16248 git::worktree_holding(&repo, &candidate.branch)
16249 .await
16250 .unwrap(),
16251 Some(registered)
16252 );
16253 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16254
16255 for _ in 0..2 {
16256 fold_run(&mut runner.state, false, &crate::run::home())
16257 .await
16258 .unwrap();
16259 runner.state = RunState::load(&runner.state.id).unwrap();
16260 assert!(runner.state.candidates[0].folded);
16261 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16262 assert_eq!(
16263 git::worktree_holding(&repo, &candidate.branch)
16264 .await
16265 .unwrap(),
16266 None
16267 );
16268 assert!(!runner.state.candidates[1].folded);
16269 assert!(winner.worktree.exists());
16270 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16271 }
16272 }
16273
16274 #[tokio::test]
16275 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16276 ask_test_home();
16277 let tmp = tempfile::tempdir().unwrap();
16278 let repo = tmp.path().join("repo");
16279 std::fs::create_dir(&repo).unwrap();
16280 init_repo(&repo);
16281 let mut runner = runner_at(RunStatus::Implementing);
16282 runner.state.repo = repo.clone();
16283 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16284 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16285 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16286 let branch = format!("candidate-{label}");
16287 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16288 git::worktree_add_branch(&repo, &wt, &branch, "main")
16289 .await
16290 .unwrap();
16291 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16292 git::git(&wt, &["add", "README.md"]).await.unwrap();
16293 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16294 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16295 runner.state.candidates.push(Candidate {
16296 index,
16297 label,
16298 agent: "alpha".to_owned(),
16299 branch,
16300 worktree: wt,
16301 summary: "completed implementation".to_owned(),
16302 stat: String::new(),
16303 files: 0,
16304 commits: 0,
16305 empty: false,
16306 failed: None,
16307 verified_noop: None,
16308 duration_ms: 42,
16309 folded: false,
16310 });
16311 runner
16312 .state
16313 .candidate_preservation
16314 .insert(label, "awaiting persistence".to_owned());
16315 }
16316 let wt = runner.state.candidates[0].worktree.clone();
16317 let index_lock = git::git(
16318 &wt,
16319 &[
16320 "rev-parse",
16321 "--path-format=absolute",
16322 "--git-path",
16323 "index.lock",
16324 ],
16325 )
16326 .await
16327 .unwrap();
16328 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16329 runner.state.save().unwrap();
16330 assert!(runner.persist_candidates().await.is_err());
16331 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16332 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16333 assert_eq!(
16334 runner.state.candidates[1].commits, 1,
16335 "other results are still collected"
16336 );
16337 assert!(
16338 !runner.state.candidates[0].empty,
16339 "failed rescue is not no change"
16340 );
16341 assert_eq!(
16342 runner.state.candidates[0].summary,
16343 "completed implementation"
16344 );
16345 assert!(runner.after_implement().is_err());
16346 let home = tmp.path().join("fold-home");
16347 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16348 assert!(wt.join("new-file").exists());
16349 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16350 assert!(recovery.join("staged.patch").exists());
16351 assert!(recovery.join("unstaged.patch").exists());
16352 assert_eq!(
16353 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16354 "untracked edits\n"
16355 );
16356 runner.state.begin_pass();
16357 runner.state.end_pass_parked(ParkCause::Upgrade {
16358 from: "1.0".into(),
16359 to: Some("1.1".into()),
16360 });
16361 runner.state.save().unwrap();
16362 runner.state = RunState::load(&runner.state.id).unwrap();
16363 assert!(matches!(
16364 runner.state.passes.last().unwrap().end,
16365 Some(crate::run::PassEnd::Parked {
16366 cause: ParkCause::Upgrade { .. }
16367 })
16368 ));
16369 std::fs::remove_file(index_lock).unwrap();
16370 runner.implement().await.unwrap();
16372 assert!(runner.state.candidate_preservation.is_empty());
16373 assert!(
16374 runner
16375 .state
16376 .candidates
16377 .iter()
16378 .all(|c| c.commits == 1 && !c.empty)
16379 );
16380 assert!(git::is_clean(&wt).await.unwrap());
16381 assert!(
16382 crate::run::read_artifact(&runner.state, "cand-A.patch")
16383 .unwrap()
16384 .contains("later tracked edits")
16385 );
16386 }
16387
16388 #[tokio::test]
16389 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16390 ask_test_home();
16391 let tmp = tempfile::tempdir().unwrap();
16392 let repo = tmp.path().join("repo");
16393 std::fs::create_dir(&repo).unwrap();
16394 init_repo(&repo);
16395 let wt = tmp.path().join("candidate");
16396 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16397 .await
16398 .unwrap();
16399 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16400 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16401 assert!(wt.join("uncommitted").exists());
16402 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16403 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16404 let mut runner = runner_at(RunStatus::Implementing);
16405 runner.state.repo = repo.clone();
16406 runner.state.base_commit = "missing-base".to_owned();
16407 runner.state.candidates.push(Candidate {
16408 index: 0,
16409 label: 'A',
16410 agent: "alpha".into(),
16411 branch: "candidate".into(),
16412 worktree: wt.clone(),
16413 summary: "completed".into(),
16414 stat: String::new(),
16415 files: 0,
16416 commits: 0,
16417 empty: false,
16418 failed: None,
16419 verified_noop: None,
16420 duration_ms: 1,
16421 folded: false,
16422 });
16423 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16424 assert!(
16425 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16426 .await
16427 .is_err()
16428 );
16429 assert!(!runner.state.candidates[0].folded);
16430 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16431 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16432 runner
16433 .state
16434 .candidate_preservation
16435 .insert('A', "pending".into());
16436 assert!(runner.persist_candidates().await.is_err());
16437 assert!(!runner.state.candidates[0].empty);
16438 assert!(!runner.state.candidates[0].folded);
16439 assert!(wt.exists());
16440 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16441 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16442 std::fs::create_dir_all(&patch_path).unwrap();
16443 assert!(
16444 runner.persist_candidates().await.is_err(),
16445 "artifact failure stays pending"
16446 );
16447 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16448 assert!(!runner.state.candidates[0].empty);
16449 std::fs::remove_dir(patch_path).unwrap();
16450 runner.persist_candidates().await.unwrap();
16451 assert!(
16452 runner.state.candidates[0].empty,
16453 "only successful inspection confirms no change"
16454 );
16455 }
16456
16457 fn init_repo(dir: &Path) {
16458 let run = |args: &[&str]| {
16459 let out = std::process::Command::new("git")
16460 .args(args)
16461 .current_dir(dir)
16462 .quiet()
16463 .output()
16464 .expect("spawn git");
16465 assert!(
16466 out.status.success(),
16467 "git {args:?} failed: {}",
16468 String::from_utf8_lossy(&out.stderr)
16469 );
16470 };
16471 run(&["init", "-b", "main"]);
16472 run(&["config", "user.name", "magi test"]);
16473 run(&["config", "user.email", "magi@example.com"]);
16474 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16475 run(&["add", "-A"]);
16476 run(&["commit", "-m", "init"]);
16477 }
16478
16479 fn ask_test_home() {
16487 crate::run::pin_test_home();
16488 }
16489
16490 #[test]
16491 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16492 ask_test_home();
16493 let mut runner = runner_at(RunStatus::Prep);
16494 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16495 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16498 let error = guard
16499 .finish(
16500 &mut runner.state,
16501 Err(
16502 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16503 .into(),
16504 ),
16505 )
16506 .unwrap_err();
16507 assert_eq!(
16508 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16509 std::io::ErrorKind::PermissionDenied
16510 );
16511 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16512 let message = format!("{error:#}");
16513 assert!(message.contains("original refusal"));
16514 assert!(!message.contains("could not save terminal"));
16515 assert_eq!(
16516 error.chain().count(),
16517 2,
16518 "only startup attribution wraps the original error"
16519 );
16520 let saved = RunState::load(&runner.state.id).unwrap();
16521 assert_eq!(saved.status, RunStatus::Blocked);
16522 assert!(!saved.startup.unwrap().complete);
16523 }
16524
16525 #[test]
16526 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16527 ask_test_home();
16528 let mut runner = runner_at(RunStatus::Prep);
16529 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16530 let id = runner.state.id.clone();
16531 drop(guard);
16532 let saved = RunState::load(&id).unwrap();
16533 assert_eq!(saved.status, RunStatus::Blocked);
16534 assert!(!saved.startup.as_ref().unwrap().complete);
16535 assert!(
16536 saved
16537 .startup
16538 .unwrap()
16539 .failure
16540 .unwrap()
16541 .contains("interrupted")
16542 );
16543 assert!(Runner::resume(&id).is_err());
16544 }
16545
16546 #[test]
16547 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16548 ask_test_home();
16549 let mut runner = runner_at(RunStatus::Prep);
16550 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16551 let error = guard
16552 .finish(
16553 &mut runner.state,
16554 Err(
16555 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16556 .into(),
16557 ),
16558 )
16559 .unwrap_err();
16560 assert_eq!(
16561 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16562 std::io::ErrorKind::PermissionDenied
16563 );
16564 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16565 let mut task = crate::queue::Task::new(
16566 "review".into(),
16567 "criteria".into(),
16568 runner.state.repo.clone(),
16569 crate::queue::Source::Human,
16570 );
16571 task.record_startup_failure(failure.clone());
16572 task.record_startup_failure(failure.clone());
16573 assert_eq!(task.startup_failures.len(), 1);
16574 assert!(task.runs.is_empty());
16575 assert_eq!(task.attempts, 0);
16576 }
16577
16578 #[cfg(unix)]
16579 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16580 ask_test_home();
16581 let tmp = tempfile::tempdir().unwrap();
16582 let repo = tmp.path().join("repo");
16583 std::fs::create_dir(&repo).unwrap();
16584 init_repo(&repo);
16585 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16586 let origin = tmp.path().join("origin.git");
16587 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16588 .await
16589 .unwrap();
16590 git::git(
16591 &repo,
16592 &["remote", "add", "origin", origin.to_str().unwrap()],
16593 )
16594 .await
16595 .unwrap();
16596 git::git(&repo, &["checkout", "-b", "fix/retained"])
16597 .await
16598 .unwrap();
16599 std::fs::write(repo.join("retained"), "retained change").unwrap();
16600 git::git(&repo, &["add", "."]).await.unwrap();
16601 git::git(&repo, &["commit", "-m", "retained implementation"])
16602 .await
16603 .unwrap();
16604 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16605 git::git(&repo, &["checkout", "main"]).await.unwrap();
16606 std::fs::write(repo.join("base-change"), "new base").unwrap();
16607 git::git(&repo, &["add", "."]).await.unwrap();
16608 git::git(&repo, &["commit", "-m", "base moved"])
16609 .await
16610 .unwrap();
16611 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16612 git::git(&repo, &["checkout", "fix/retained"])
16613 .await
16614 .unwrap();
16615 let mut runner = runner_at(RunStatus::Voting);
16616 runner.state.repo = repo.clone();
16617 runner.state.base_commit = base;
16618 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16619 runner.state.config.disk.min_free_bytes = 0;
16620 runner.state.config.graph.review_rounds = 1;
16621 runner.state.config.verify.e2e = vec!["true".into()];
16622 runner.state.config.verify.gate = vec!["true".into()];
16623 runner.state.candidates = vec![Candidate {
16624 index: 0,
16625 label: 'A',
16626 agent: "fixture".into(),
16627 branch: "fix/retained".into(),
16628 worktree: repo.clone(),
16629 summary: String::new(),
16630 stat: String::new(),
16631 files: 1,
16632 commits: 1,
16633 empty: false,
16634 failed: None,
16635 verified_noop: None,
16636 duration_ms: 0,
16637 folded: false,
16638 }];
16639 runner.state.tally = Some(Tally {
16640 first_choice: BTreeMap::new(),
16641 borda: BTreeMap::new(),
16642 winner: 'A',
16643 rankings: 0,
16644 unanimous_initial: true,
16645 deliberated: false,
16646 changed_votes: 0,
16647 unanimous_final: true,
16648 tie_break: None,
16649 judges: 0,
16650 present: 0,
16651 quorum: 0,
16652 met_quorum: true,
16653 uncontested: Some("review-only".into()),
16654 });
16655 let mut task = crate::queue::Task::new(
16656 "fixture".into(),
16657 "fixture".into(),
16658 repo,
16659 crate::queue::Source::Human,
16660 );
16661 task.link_run(&runner.state.id);
16662 crate::queue::Queue::open().put(&mut task).unwrap();
16663 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16664 runner.state.base_sync = Some(BaseSync {
16665 tip: String::new(),
16666 behind: 1,
16667 attempts: BASE_SYNC_ROUNDS,
16668 conflict: None,
16669 already_in: None,
16670 });
16671 runner.sync_to_base().await.unwrap();
16672 assert_eq!(runner.state.status, RunStatus::Blocked);
16673 (tmp, runner, head, task.id)
16674 }
16675
16676 #[cfg(unix)]
16677 #[tokio::test]
16678 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16679 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16680 if std::env::var_os(CHILD).is_none() {
16681 let home = tempfile::tempdir().unwrap();
16682 let output = std::process::Command::new(std::env::current_exe().unwrap())
16683 .args([
16684 "--exact",
16685 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16686 "--nocapture",
16687 ])
16688 .env(CHILD, "1")
16689 .env("MAGI_HOME", home.path())
16690 .env("MAGI_WORKER_HOME", home.path())
16691 .env("MAGI_WORKER_ID", "abcdef")
16692 .output()
16693 .unwrap();
16694 assert!(
16695 output.status.success()
16696 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16697 "{}\n{}",
16698 String::from_utf8_lossy(&output.stdout),
16699 String::from_utf8_lossy(&output.stderr)
16700 );
16701 return;
16702 }
16703 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16704 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16705 let home = crate::run::home();
16706 let queue = crate::queue::Queue::open();
16707 let mut owner = queue.get(&task).unwrap();
16708 owner.release_operator("test-release", None);
16709 owner.start(runner.state.id.clone());
16710 queue.put(&mut owner).unwrap();
16711 runner.state.driver_exited = true;
16712 runner.state.task_instruction = Some(owner.instruction.clone());
16713 runner.state.save().unwrap();
16714 let _claim = queue.claim(&task).unwrap();
16715 let reserve = |id: &str| {
16716 let directory = home.join("workers/jobs").join(id);
16717 std::fs::create_dir_all(&directory).unwrap();
16718 let job = crate::worker::Job {
16719 id: id.into(),
16720 sequence: 1,
16721 kind: "task".into(),
16722 key: format!("task:{task}"),
16723 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16724 generation: "fixture-generation".into(),
16725 version: None,
16726 status: crate::worker::Status::Running,
16727 result: None,
16728 pid: Some(std::process::id()),
16729 created_at: 0,
16730 finished_at: None,
16731 };
16732 std::fs::write(
16733 directory.join("job.json"),
16734 serde_json::to_vec(&job).unwrap(),
16735 )
16736 .unwrap();
16737 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16738 };
16739 let _own = reserve("abcdef");
16740 let other = reserve("fedcba");
16741 assert!(
16742 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16743 .await
16744 .is_err()
16745 );
16746 assert!(
16747 RunState::load(&runner.state.id)
16748 .unwrap()
16749 .base_sync_recovery
16750 .authorizations
16751 .is_empty()
16752 );
16753 drop(other);
16754 assert!(crate::daemon::is_working_on(
16755 &home,
16756 &runner.state.id,
16757 Timestamp::now()
16758 ));
16759 assert!(
16760 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16761 );
16762 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16763 .await
16764 .unwrap();
16765 assert!(
16766 RunState::load(&runner.state.id)
16767 .unwrap()
16768 .base_recovery_pending()
16769 );
16770 }
16771
16772 #[cfg(unix)]
16773 #[tokio::test]
16774 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16775 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16776 let home = crate::run::home();
16777 runner.state.driver_pid = Some(std::process::id());
16778 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16779 runner.state.driver_exited = false;
16780 runner.state.begin_pass();
16781 runner.state.save().unwrap();
16782 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16783 driver.record(&runner.state).unwrap();
16784 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16785 drop(driver);
16787 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16788 assert_eq!(
16789 runner.state.liveness_under(&home, false),
16790 crate::run::Liveness::Dead
16791 );
16792 let queue = crate::queue::Queue::open();
16793 let _claim = queue.claim(&task).unwrap();
16794 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16795 .await
16796 .unwrap();
16797 let disk = RunState::load(&runner.state.id).unwrap();
16798 assert!(!disk.driver_exited);
16799 assert!(disk.base_recovery_pending());
16800 }
16801
16802 #[cfg(unix)]
16803 #[tokio::test]
16804 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16805 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16806 if std::env::var_os(CHILD).is_none() {
16807 use std::os::unix::fs::PermissionsExt;
16808 let tmp = tempfile::tempdir().unwrap();
16809 let gh = tmp.path().join("gh");
16810 std::fs::write(&gh, r#"#!/bin/sh
16811case "$1 $2" in
16812 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16813 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16814 *) exit 2 ;;
16815esac
16816"#).unwrap();
16817 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16818 let mut paths = vec![tmp.path().to_path_buf()];
16819 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16820 let output = std::process::Command::new(std::env::current_exe().unwrap())
16821 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16822 .env(CHILD, "1")
16823 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16824 .env("PATH", std::env::join_paths(paths).unwrap())
16825 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16826 .output().unwrap();
16827 assert!(
16828 output.status.success()
16829 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16830 "{}\n{}",
16831 String::from_utf8_lossy(&output.stdout),
16832 String::from_utf8_lossy(&output.stderr)
16833 );
16834 return;
16835 }
16836 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16837 let repo = runner.state.repo.clone();
16838 git::git(&repo, &["checkout", "main"]).await.unwrap();
16839 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16840 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16841 git::git(&repo, &["checkout", "fix/retained"])
16842 .await
16843 .unwrap();
16844 runner.state.pr = Some(crate::run::PrRecord {
16845 url: "https://example.test/project/pull/1".into(),
16846 number: 1,
16847 state: "open".into(),
16848 checks: "unknown".into(),
16849 round: 0,
16850 rounds: 1,
16851 red_at_merge: Vec::new(),
16852 });
16853 runner.sync_to_base().await.unwrap();
16854 assert_eq!(runner.state.status, RunStatus::Blocked);
16855 assert_eq!(
16856 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16857 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16858 );
16859 assert!(runner.state.base_sync_closure_pending());
16860 assert!(crate::handover::can_resume(&runner.state));
16861 let queue = crate::queue::Queue::open();
16862 assert!(!crate::base_sync_recovery::held_task(
16863 &queue,
16864 &queue.get(&task).unwrap()
16865 ));
16866 assert!(runner.state.authorize_base_sync(&task).is_err());
16867 runner.state.base_sync_recovery.stop = None;
16869 runner.state.status = RunStatus::Voting;
16870 runner.state.gate_ran = true;
16871 runner.state.config.graph.review_rounds = 0;
16872 runner.state.save().unwrap();
16873 assert!(crate::handover::can_resume(&runner.state));
16874 runner.state = RunState::load(&runner.state.id).unwrap();
16875 runner.execute_graph().await.unwrap();
16876 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16877 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16878 assert_eq!(
16879 runner.state.base_sync.as_ref().unwrap().attempts,
16880 BASE_SYNC_ROUNDS
16881 );
16882 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16883 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16884 assert!(runner.state.reviews.is_empty());
16885 assert!(runner.state.gate.is_empty());
16886 }
16887
16888 #[cfg(unix)]
16889 #[tokio::test]
16890 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16891 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16892 runner.state.driver_exited = true;
16893 runner.state.save().unwrap();
16894 let home = crate::run::home();
16895 let queue = crate::queue::Queue::open();
16896 let _claim = queue.claim(&task).unwrap();
16897 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16898 assert!(
16899 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16900 .await
16901 .is_err()
16902 );
16903 drop(lease);
16904 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16905 assert!(
16906 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16907 .await
16908 .is_err()
16909 );
16910 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16911 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16912 .await
16913 .unwrap();
16914 let disk = RunState::load(&runner.state.id).unwrap();
16915 assert!(disk.base_recovery_pending());
16916 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16917 assert!(
16918 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16919 .await
16920 .is_err()
16921 );
16922 assert_eq!(
16923 RunState::load(&runner.state.id)
16924 .unwrap()
16925 .base_sync_recovery
16926 .authorizations
16927 .len(),
16928 1
16929 );
16930 let mut owner = queue.get(&task).unwrap();
16931 owner.runs.push("20260101-000000-ffff".into());
16932 queue.put(&mut owner).unwrap();
16933 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16934 .await
16935 .unwrap_err();
16936 assert!(error.to_string().contains("run task ownership changed"));
16937 }
16938
16939 #[cfg(unix)]
16940 #[tokio::test]
16941 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16942 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16943 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16944 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16945 legacy["status"] = serde_json::json!("voting");
16946 runner.state = serde_json::from_value(legacy).unwrap();
16947 runner.state.save().unwrap();
16948 for _ in 0..3 {
16949 runner.execute_graph().await.unwrap();
16950 assert_eq!(runner.state.status, RunStatus::Blocked);
16951 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16952 assert!(!crate::handover::can_resume(&runner.state));
16953 assert_eq!(
16954 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16955 head
16956 );
16957 assert!(runner.state.reviews.is_empty());
16958 assert!(!runner.state.gate_ran);
16959 }
16960 let notice = crate::notices::run_ended(&runner.state).unwrap();
16961 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16962 runner.state.begin_pass();
16963 runner.state.status = RunStatus::Voting;
16964 runner.deliberate().await.unwrap();
16965 runner.vote().await.unwrap();
16966 runner.finish_after_tally().await.unwrap();
16967 assert_eq!(runner.state.status, RunStatus::Blocked);
16968 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16969 assert_eq!(
16970 crate::notices::run_ended(&runner.state).unwrap().since,
16971 notice.since
16972 );
16973 }
16974
16975 #[cfg(unix)]
16976 #[tokio::test]
16977 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16978 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16979 for detail in [
16980 "fix/retained conflicts with origin/main and did not rebase: conflict",
16981 "rebased fix/retained locally but origin refused the push",
16982 "unknown failure",
16983 ] {
16984 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16985 runner.state.record_base_sync_stop(&head);
16986 assert!(runner.state.authorize_base_sync(&task).is_err());
16987 runner.state.status = RunStatus::Voting;
16988 runner.execute_graph().await.unwrap();
16989 assert_eq!(runner.state.status, RunStatus::Blocked);
16990 assert!(!crate::handover::can_resume(&runner.state));
16991 assert_eq!(
16992 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16993 head
16994 );
16995 }
16996 }
16997
16998 #[cfg(unix)]
16999 #[tokio::test]
17000 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17001 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17002 runner.state.authorize_base_sync(&task).unwrap();
17003 assert!(runner.state.consume_base_recovery(&head));
17004 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17005 runner.state.reserve_base_operation(&head, &tip);
17006 runner.state.save().unwrap();
17007 runner.state = RunState::load(&runner.state.id).unwrap();
17008 assert!(!runner.state.consume_base_recovery(&head));
17009 runner.sync_to_base().await.unwrap();
17010 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17011 assert_eq!(runner.state.base_episode_used(), 1);
17012 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17013 assert!(
17014 runner
17015 .state
17016 .base_sync_recovery
17017 .operations
17018 .last()
17019 .unwrap()
17020 .finished
17021 );
17022 assert!(!runner.state.base_sync_blocked());
17023 runner.sync_to_base().await.unwrap();
17024 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17025 }
17026
17027 #[cfg(unix)]
17028 #[tokio::test]
17029 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17030 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17031 let original = runner
17033 .state
17034 .base_sync
17035 .as_ref()
17036 .unwrap()
17037 .conflict
17038 .clone()
17039 .unwrap();
17040 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17041 original
17042 .replace("synchronization(s)", "rebase(s)")
17043 .replace("synchronizing", "rebasing"),
17044 );
17045 runner.state.base_sync_recovery = Default::default();
17046 runner.state.record_base_sync_stop(&head);
17047 assert_eq!(
17048 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17049 crate::base_sync_recovery::StopKind::Exhausted
17050 );
17051 runner.state.authorize_base_sync(&task).unwrap();
17052 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17053 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17054 .as_object_mut()
17055 .unwrap();
17056 auth.remove("interval");
17057 auth.remove("scope");
17058 runner.state = serde_json::from_value(legacy).unwrap();
17059 assert!(runner.state.consume_base_recovery(&head));
17060 assert_eq!(runner.state.base_episode_used(), 3);
17061 runner.sync_to_base().await.unwrap();
17062 assert_eq!(runner.state.base_episode_used(), 4);
17063 runner.state.resource_block = Some(crate::run::ResourceBlock {
17064 node: "gate".into(),
17065 kind: "cache".into(),
17066 detail: "legacy wait".into(),
17067 at: Timestamp::now(),
17068 cause: crate::run::ResourceBlockCause::Unrecorded,
17069 });
17070 runner.state.save().unwrap();
17071 assert!(
17072 runner
17073 .state
17074 .base_sync_recovery
17075 .episodes
17076 .last()
17077 .unwrap()
17078 .boundary
17079 .is_none()
17080 );
17081 runner.state.resource_block = None;
17082 advance_base_fixture(&runner).await;
17083 runner.sync_to_base().await.unwrap();
17084 assert!(runner.state.base_sync_blocked());
17085 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17086 }
17087
17088 #[cfg(unix)]
17089 async fn advance_base_fixture(runner: &Runner) {
17090 let repo = &runner.state.repo;
17091 for args in [
17092 vec!["checkout", "main"],
17093 vec!["commit", "--allow-empty", "-m", "base advanced"],
17094 vec!["push", "origin", "main"],
17095 vec!["checkout", "fix/retained"],
17096 ] {
17097 git::git(repo, &args).await.unwrap();
17098 }
17099 }
17100
17101 #[cfg(unix)]
17102 fn fresh_base_interval_fixture(runner: &mut Runner) {
17103 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17104 runner.state.base_sync.as_mut().unwrap().conflict = None;
17105 runner.state.base_sync_recovery = Default::default();
17106 runner.state.status = RunStatus::Voting;
17107 }
17108
17109 #[cfg(unix)]
17110 #[tokio::test]
17111 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17112 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17113 fresh_base_interval_fixture(&mut runner);
17114 runner.state.begin_pass();
17115 runner.sync_to_base().await.unwrap();
17116 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17117 runner.state.resource_block = Some(crate::run::ResourceBlock {
17118 node: "gate".into(),
17119 kind: "build cache".into(),
17120 detail: "busy".into(),
17121 at: Timestamp::now(),
17122 cause: crate::run::ResourceBlockCause::Timeout {
17123 waited_secs: 1,
17124 limit_secs: 1,
17125 },
17126 });
17127 runner.state.save().unwrap();
17128 let queue = crate::queue::Queue::open();
17129 let mut owner = queue.get(&task).unwrap();
17130 runner.state.task_instruction = Some(owner.instruction.clone());
17131 owner.release_operator("fixture-owner", None);
17132 queue.put(&mut owner).unwrap();
17133 let approval = owner.resumption.clone().unwrap();
17134 runner.state.resource_block = None;
17135 runner.state.save().unwrap();
17136 runner.state = RunState::load(&runner.state.id).unwrap();
17137 advance_base_fixture(&runner).await;
17138 runner.sync_to_base().await.unwrap();
17139 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17140 assert_eq!(runner.state.base_episode_used(), 1);
17141 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17142 assert!(
17143 runner.state.base_sync_recovery.episodes[0]
17144 .boundary
17145 .as_ref()
17146 .unwrap()
17147 .consumed
17148 );
17149 assert!(
17150 !runner
17151 .state
17152 .renew_base_episode(&task, &approval, &head, "base")
17153 );
17154 for _ in 0..3 {
17155 owner.release_operator("fixture-owner", None);
17156 queue.put(&mut owner).unwrap();
17157 runner.state = RunState::load(&runner.state.id).unwrap();
17158 runner.state.begin_pass();
17159 runner.sync_to_base().await.unwrap();
17160 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17161 assert_eq!(runner.state.base_episode_used(), 1);
17162 }
17163 runner.state.resource_block = Some(crate::run::ResourceBlock {
17164 node: "gate".into(),
17165 kind: "disk pressure".into(),
17166 detail: "free space required".into(),
17167 at: Timestamp::now(),
17168 cause: crate::run::ResourceBlockCause::Other,
17169 });
17170 runner.state.end_pass_parked(ParkCause::Interrupt {
17171 reason: "disk pressure".into(),
17172 });
17173 runner.state.save().unwrap();
17174 owner.release_operator("fixture-owner-after-disk", None);
17175 queue.put(&mut owner).unwrap();
17176 runner.state.resource_block = None;
17177 advance_base_fixture(&runner).await;
17178 runner.sync_to_base().await.unwrap();
17179 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17180 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17181 assert_eq!(runner.state.base_episode_used(), 1);
17182 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17183 }
17184
17185 #[cfg(unix)]
17186 #[tokio::test]
17187 async fn base_recovery_continuous_advancement_stops_at_four() {
17188 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17189 fresh_base_interval_fixture(&mut runner);
17190 for spent in 1..=BASE_SYNC_ROUNDS {
17191 runner.sync_to_base().await.unwrap();
17192 assert!(!runner.state.base_sync_blocked());
17193 assert_eq!(runner.state.base_episode_used(), spent);
17194 advance_base_fixture(&runner).await;
17195 }
17196 runner.sync_to_base().await.unwrap();
17197 assert_eq!(
17198 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17199 crate::base_sync_recovery::StopKind::Exhausted
17200 );
17201 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17202 assert!(stop.unverified_sync);
17203 let action = runner.state.base_sync_next_action();
17204 assert!(action.contains("never reached its gate"));
17205 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17206 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17207 runner.state = RunState::load(&runner.state.id).unwrap();
17208 runner.sync_to_base().await.unwrap();
17209 assert_eq!(runner.state.base_episode_used(), 4);
17210 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17211 }
17212
17213 #[cfg(unix)]
17214 #[tokio::test]
17215 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17216 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17217 runner.state.ensure_base_episode();
17218 runner.state.record_base_progress(&head);
17220 runner.state.base_sync_recovery.stop = None;
17221 runner.state.record_base_sync_stop(&head);
17222 assert!(
17223 !runner
17224 .state
17225 .base_sync_recovery
17226 .stop
17227 .as_ref()
17228 .unwrap()
17229 .unverified_sync
17230 );
17231
17232 runner.state.finish_base_operation(Some(&head));
17233 runner.state.gate_ran = true;
17235 runner.state.gate_head = Some("older-head".into());
17236 runner.state.base_sync_recovery.stop = None;
17237 runner.state.record_base_sync_stop(&head);
17238 runner.state.save().unwrap();
17239 runner.state = RunState::load(&runner.state.id).unwrap();
17240 assert!(
17241 runner
17242 .state
17243 .base_sync_recovery
17244 .stop
17245 .as_ref()
17246 .unwrap()
17247 .unverified_sync
17248 );
17249
17250 runner.state.gate_head = Some(head.clone());
17251 runner.state.base_sync_recovery.stop = None;
17252 runner.state.record_base_sync_stop(&head);
17253 assert!(
17254 !runner
17255 .state
17256 .base_sync_recovery
17257 .stop
17258 .as_ref()
17259 .unwrap()
17260 .unverified_sync
17261 );
17262 assert!(
17263 !runner
17264 .state
17265 .base_sync_next_action()
17266 .contains("never reached its gate")
17267 );
17268
17269 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17271 legacy["base_sync_recovery"]["stop"]
17272 .as_object_mut()
17273 .unwrap()
17274 .remove("unverified_sync");
17275 legacy["base_sync_recovery"]["episodes"][0]
17276 .as_object_mut()
17277 .unwrap()
17278 .remove("last_sync_head");
17279 runner.state = serde_json::from_value(legacy).unwrap();
17280 assert!(
17281 !runner
17282 .state
17283 .base_sync_recovery
17284 .stop
17285 .as_ref()
17286 .unwrap()
17287 .unverified_sync
17288 );
17289 assert!(
17290 runner.state.base_sync_recovery.episodes[0]
17291 .last_sync_head
17292 .is_none()
17293 );
17294 }
17295
17296 #[cfg(unix)]
17297 #[tokio::test]
17298 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17299 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17300 runner.state.authorize_base_sync(&task).unwrap();
17301 assert!(
17302 runner
17303 .state
17304 .base_sync_next_action()
17305 .contains("Approval recorded")
17306 );
17307 assert!(
17308 runner
17309 .state
17310 .base_sync_next_action()
17311 .contains("4/4 in episode 0, 4 lifetime")
17312 );
17313 assert!(runner.state.consume_base_recovery(&head));
17314 assert!(
17315 runner
17316 .state
17317 .base_sync_next_action()
17318 .contains("0/4 in episode 1, 4 lifetime")
17319 );
17320 }
17321
17322 #[cfg(unix)]
17323 #[tokio::test]
17324 async fn base_recovery_changed_head_invalidates_without_spending() {
17325 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17326 runner.state.authorize_base_sync(&task).unwrap();
17327 git::git(
17328 &runner.state.repo,
17329 &["commit", "--allow-empty", "-m", "external head change"],
17330 )
17331 .await
17332 .unwrap();
17333 runner.sync_to_base().await.unwrap();
17334 let a = runner
17335 .state
17336 .base_sync_recovery
17337 .authorizations
17338 .last()
17339 .unwrap();
17340 assert!(a.consumed);
17341 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17342 runner.state = RunState::load(&runner.state.id).unwrap();
17343 assert!(
17344 runner
17345 .state
17346 .base_sync_next_action()
17347 .contains("Approval invalidated")
17348 );
17349 assert!(!runner.state.base_recovery_pending());
17350 assert!(runner.state.base_sync_blocked());
17351 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17352 }
17353
17354 #[cfg(unix)]
17355 #[tokio::test]
17356 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17357 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17358 runner.state.driver_exited = true;
17359 runner.state.save().unwrap();
17360 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17361 .await
17362 .unwrap();
17363 runner.state = RunState::load(&runner.state.id).unwrap();
17364 advance_base_fixture(&runner).await;
17365 runner.sync_to_base().await.unwrap();
17366 let a = runner
17367 .state
17368 .base_sync_recovery
17369 .authorizations
17370 .last()
17371 .unwrap();
17372 assert!(
17373 a.consumed
17374 && a.invalidated
17375 .as_deref()
17376 .unwrap()
17377 .contains("fetched base changed")
17378 );
17379 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17380 assert_eq!(
17381 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17382 head
17383 );
17384 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17385 .await
17386 .unwrap();
17387 runner.state = RunState::load(&runner.state.id).unwrap();
17388 runner.sync_to_base().await.unwrap();
17389 assert!(!runner.state.base_sync_blocked());
17390 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17391 }
17392
17393 #[cfg(unix)]
17394 #[tokio::test]
17395 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17396 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17397 runner.state.authorize_base_sync(&task).unwrap();
17398 assert!(runner.state.consume_base_recovery(&head));
17399 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17400 runner.state.reserve_base_operation(&head, &tip);
17401 runner.state.save().unwrap();
17402 let scratch = runner.state.dir().join("base-sync");
17403 assert_eq!(
17404 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17405 .await
17406 .unwrap(),
17407 crate::rebase::Rebased::Applied
17408 );
17409 runner.state = RunState::load(&runner.state.id).unwrap();
17410 runner.sync_to_base().await.unwrap();
17411 assert!(
17412 runner
17413 .state
17414 .base_sync_recovery
17415 .operations
17416 .last()
17417 .unwrap()
17418 .finished
17419 );
17420 assert_eq!(runner.state.base_episode_used(), 1);
17421 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17422 assert!(!runner.state.base_sync_blocked());
17423 }
17424
17425 #[cfg(unix)]
17426 #[tokio::test]
17427 async fn base_recovery_crash_preserves_untracked_files() {
17428 for ignored in [false, true] {
17429 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17430 runner.state.authorize_base_sync(&task).unwrap();
17431 assert!(runner.state.consume_base_recovery(&head));
17432 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17433 runner.state.reserve_base_operation(&head, &tip);
17434 runner.state.save().unwrap();
17435 let scratch = runner.state.dir().join("base-sync");
17436 assert_eq!(
17437 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17438 .await
17439 .unwrap(),
17440 crate::rebase::Rebased::Applied
17441 );
17442 let repo = &runner.state.repo;
17443 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17444 assert_ne!(result, head);
17445 if ignored {
17446 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17447 .await
17448 .unwrap();
17449 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17450 }
17451 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17452 assert!(
17453 git::git_raw(repo, &["diff", "--quiet", &head])
17454 .await
17455 .unwrap()
17456 .ok()
17457 );
17458 runner.state = RunState::load(&runner.state.id).unwrap();
17460 runner.sync_to_base().await.unwrap();
17461 assert!(runner.state.base_sync_blocked());
17462 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17463 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17464 assert_eq!(
17465 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17466 result
17467 );
17468 assert_eq!(
17469 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17470 "retained work\n"
17471 );
17472 assert!(
17473 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17474 .await
17475 .unwrap()
17476 .ok()
17477 );
17478 }
17479 }
17480
17481 #[cfg(unix)]
17482 #[tokio::test]
17483 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17484 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17485 runner.state.authorize_base_sync(&task).unwrap();
17486 assert!(runner.state.consume_base_recovery(&head));
17487 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17488 runner.state.reserve_base_operation(&head, &tip);
17489 runner.state.save().unwrap();
17490 let scratch = runner.state.dir().join("base-sync");
17491 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17492 .await
17493 .unwrap();
17494 git::sync_to_head(&runner.state.repo).await.unwrap();
17495 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17496 git::git(&runner.state.repo, &["add", "foreign"])
17497 .await
17498 .unwrap();
17499 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17500 .await
17501 .unwrap();
17502 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17503 runner.state = RunState::load(&runner.state.id).unwrap();
17504 runner.sync_to_base().await.unwrap();
17505 assert!(runner.state.base_sync_blocked());
17506 assert!(
17507 !runner
17508 .state
17509 .base_sync_recovery
17510 .operations
17511 .last()
17512 .unwrap()
17513 .finished
17514 );
17515 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17516 assert_eq!(
17517 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17518 foreign_head
17519 );
17520 }
17521
17522 #[cfg(unix)]
17523 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17524 {
17525 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17526 fresh_base_interval_fixture(&mut runner);
17527 let repo = &runner.state.repo;
17528 git::git(repo, &["checkout", "main"]).await.unwrap();
17529 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17530 git::git(repo, &["add", "retained"]).await.unwrap();
17531 git::git(repo, &["commit", "-m", "upstream retained change"])
17532 .await
17533 .unwrap();
17534 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17535 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17536 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17537 let scratch = runner.state.dir().join("base-sync");
17538 assert!(matches!(
17539 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17540 .await
17541 .unwrap(),
17542 git::RebaseStart::Conflicted(_)
17543 ));
17544 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17545 agent: "fixture".into(),
17546 paths: vec!["retained".into()],
17547 from: Some(head.clone()),
17548 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17549 index_outside: None,
17550 finished: false,
17551 error: None,
17552 launcher: None,
17553 infra: None,
17554 outage: None,
17555 });
17556 (tmp, runner, head, tip, scratch)
17557 }
17558
17559 #[cfg(unix)]
17560 #[tokio::test]
17561 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17562 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17563 std::fs::write(
17566 scratch.join("retained"),
17567 "upstream and retained resolution\n",
17568 )
17569 .unwrap();
17570 runner.state.resource_block = Some(crate::run::ResourceBlock {
17571 node: "rebase".into(),
17572 kind: "repository coordination".into(),
17573 detail: "busy".into(),
17574 at: Timestamp::now(),
17575 cause: crate::run::ResourceBlockCause::Other,
17576 });
17577 runner.state.save().unwrap();
17578 let newer = git::git(
17581 &runner.state.repo,
17582 &[
17583 "commit-tree",
17584 "main^{tree}",
17585 "-p",
17586 "main",
17587 "-m",
17588 "base advanced during wait",
17589 ],
17590 )
17591 .await
17592 .unwrap();
17593 git::git(
17594 &runner.state.repo,
17595 &[
17596 "push",
17597 "origin",
17598 &format!("{}:refs/heads/main", newer.trim()),
17599 ],
17600 )
17601 .await
17602 .unwrap();
17603 runner.state = RunState::load(&runner.state.id).unwrap();
17604 runner.state.resource_block = None;
17605 runner.sync_to_base().await.unwrap();
17606 assert!(!runner.state.base_sync_blocked());
17607 let op = &runner.state.base_sync_recovery.operations[0];
17608 assert_eq!(op.head, head);
17609 assert_eq!(op.tip, tip);
17610 assert!(op.finished);
17611 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17612 assert_eq!(runner.state.rebase_fixes.len(), 1);
17613 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17614 .await
17615 .unwrap();
17616 assert_ne!(latest, tip);
17617 runner.state = RunState::load(&runner.state.id).unwrap();
17618 runner.sync_to_base().await.unwrap();
17619 assert!(!runner.state.base_sync_blocked());
17620 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17621 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17622 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17623 }
17624
17625 #[cfg(unix)]
17626 #[tokio::test]
17627 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17628 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17629 runner.state.reserve_base_operation(&head, &tip);
17630 runner.state.save().unwrap();
17631 std::fs::write(
17632 scratch.join("retained"),
17633 "upstream and retained resolution\n",
17634 )
17635 .unwrap();
17636 git::git(&scratch, &["add", "retained"]).await.unwrap();
17637 git::git(
17638 &scratch,
17639 &["-c", "core.editor=true", "rebase", "--continue"],
17640 )
17641 .await
17642 .unwrap();
17643 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17644 assert!(
17645 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17646 .await
17647 .unwrap()
17648 .is_empty()
17649 );
17650 assert_eq!(
17651 crate::rebase::finish(
17652 &mut runner.state,
17653 &scratch,
17654 "fix/retained",
17655 &head,
17656 &tip,
17657 &["retained".into()],
17658 ""
17659 )
17660 .await
17661 .unwrap(),
17662 crate::rebase::Rebased::Applied
17663 );
17664 assert!(!scratch.exists());
17665 runner.state = RunState::load(&runner.state.id).unwrap();
17667 let op = &runner.state.base_sync_recovery.operations[0];
17668 assert!(!op.finished);
17669 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17670 runner.sync_to_base().await.unwrap();
17671 assert!(!runner.state.base_sync_blocked());
17672 assert!(runner.state.base_sync_recovery.operations[0].finished);
17673 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17674 assert_eq!(runner.state.rebase_fixes.len(), 1);
17675 let winner = runner.state.winner().unwrap().clone();
17676 let wait = runner.repository_wait();
17677 assert!(
17678 verify_current_head(&mut runner.state, &winner, &wait)
17679 .await
17680 .unwrap()
17681 );
17682 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17683 }
17684
17685 #[cfg(unix)]
17686 #[tokio::test]
17687 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17688 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17689 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17690 runner.state.reserve_base_operation(&head, &tip);
17691 runner.state.save().unwrap();
17692 std::fs::write(
17693 scratch.join("retained"),
17694 "upstream and retained resolution\n",
17695 )
17696 .unwrap();
17697 git::git(&scratch, &["add", "retained"]).await.unwrap();
17698 git::git(
17699 &scratch,
17700 &["-c", "core.editor=true", "rebase", "--continue"],
17701 )
17702 .await
17703 .unwrap();
17704 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17705 assert!(!git::rebase_in_progress(&scratch).await);
17706 assert!(
17707 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17708 .await
17709 .unwrap()
17710 .is_empty()
17711 );
17712 match damage {
17713 Some("record") => {
17714 let path =
17715 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17716 .await
17717 .unwrap();
17718 std::fs::write(scratch.join(path), "").unwrap();
17719 }
17720 Some("foreign") => {
17721 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17722 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17723 git::git(&scratch, &["commit", "-m", "foreign work"])
17724 .await
17725 .unwrap();
17726 }
17727 Some("dirty") => {
17728 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17729 }
17730 _ => {}
17731 }
17732 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17733 runner.state = RunState::load(&runner.state.id).unwrap();
17735 assert!(
17736 runner.state.base_sync_recovery.operations[0]
17737 .validated_head
17738 .is_none()
17739 );
17740 runner.sync_to_base().await.unwrap();
17741 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17742 assert_eq!(runner.state.base_episode_used(), 1);
17743 assert_eq!(runner.state.rebase_fixes.len(), 1);
17744 assert_eq!(
17745 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17746 retained_head
17747 );
17748 if damage.is_some() {
17749 assert!(runner.state.base_sync_blocked());
17750 assert!(scratch.exists());
17751 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17752 assert!(
17753 runner.state.base_sync_recovery.operations[0]
17754 .validated_head
17755 .is_none()
17756 );
17757 } else {
17758 assert!(!runner.state.base_sync_blocked());
17759 assert!(!scratch.exists());
17760 assert!(runner.state.base_sync_recovery.operations[0].finished);
17761 assert_eq!(
17762 runner.state.base_sync_recovery.operations[0]
17763 .validated_head
17764 .as_deref(),
17765 Some(result.as_str())
17766 );
17767 let winner = runner.state.winner().unwrap().clone();
17768 let wait = runner.repository_wait();
17769 assert!(
17770 verify_current_head(&mut runner.state, &winner, &wait)
17771 .await
17772 .unwrap()
17773 );
17774 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17775 runner.state = RunState::load(&runner.state.id).unwrap();
17776 runner.sync_to_base().await.unwrap();
17777 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17778 }
17779 }
17780 }
17781
17782 #[cfg(unix)]
17783 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17784 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17785 runner.state.prep_complete = Some(true);
17787 runner.state.base_sync = None;
17788 runner.state.base_sync_recovery = Default::default();
17789 runner.state.config.graph.gate_fix_rounds = 1;
17790 let mut review = review_round(true, 0, 1, 1, false, true);
17791 review.head = head.clone();
17792 review.verified_head = Some(head.clone());
17793 runner.state.reviews = vec![review];
17794 runner.state.gate = vec![CommandOutcome {
17795 command: "old gate".into(),
17796 code: Some(1),
17797 output_tail: "old failure".into(),
17798 duration_ms: 0,
17799 resource_blocked: false,
17800 }];
17801 runner.state.gate_head = Some(head.clone());
17802 runner.state.gate_ran = true;
17803 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17804 agent: "fixture".into(),
17805 failed: runner.state.gate.clone(),
17806 notes: "spent".into(),
17807 committed: false,
17808 error: None,
17809 }];
17810 runner.state.land_approval = Some(crate::run::LandApproval {
17811 question: "old approval".into(),
17812 head: head.clone(),
17813 });
17814 runner.state.status = RunStatus::Blocked;
17815 (tmp, runner, head)
17816 }
17817
17818 #[cfg(unix)]
17819 #[tokio::test]
17820 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17821 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17822 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17823 let phases = tmp.path().join("phases");
17824 runner.state.config.verify.e2e = vec![format!(
17825 "echo e2e >> '{}'; {}",
17826 phases.display(),
17827 if e2e_ok {
17828 "true"
17829 } else {
17830 "echo new-e2e-failure; false"
17831 }
17832 )];
17833 runner.state.config.verify.gate = vec![format!(
17834 "echo gate >> '{}'; {}",
17835 phases.display(),
17836 if gate_ok {
17837 "true"
17838 } else {
17839 "echo new-gate-failure; false"
17840 }
17841 )];
17842 runner.state.save().unwrap();
17843 runner.state = RunState::load(&runner.state.id).unwrap();
17844 runner.sync_to_base().await.unwrap();
17845 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17846 assert_ne!(head, old);
17847 runner.state = RunState::load(&runner.state.id).unwrap();
17849 runner.execute_graph().await.unwrap();
17850 assert_eq!(runner.state.gate_fixes.len(), 1);
17851 assert_eq!(runner.state.reviews.len(), 1);
17852 assert_eq!(runner.state.reviews[0].head, old);
17853 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17854 assert_eq!(runner.state.verification_history.len(), 1);
17855 let history = &runner.state.verification_history[0];
17856 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17857 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17858 assert!(history.archived_at.is_some());
17859 assert_eq!(history.fixes_spent, Some(1));
17860 assert_eq!(history.gate[0].output_tail, "old failure");
17861 let archive_events: Vec<_> = runner
17862 .state
17863 .events
17864 .iter()
17865 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17866 .collect();
17867 assert_eq!(archive_events.len(), 1);
17868 assert!(archive_events[0].message.contains(&short(&old)));
17869 assert!(archive_events[0].message.contains(&short(&head)));
17870 assert!(
17871 archive_events[0]
17872 .message
17873 .contains("1 gate-fix round(s) already spent stay spent")
17874 );
17875 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17876 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17877 if e2e_ok {
17878 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17879 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17880 } else {
17881 assert!(!runner.state.gate_ran);
17882 }
17883 assert_eq!(
17884 runner.state.status,
17885 if e2e_ok && gate_ok {
17886 RunStatus::Ready
17887 } else {
17888 RunStatus::Blocked
17889 }
17890 );
17891 if !e2e_ok || !gate_ok {
17892 let cause = runner.state.verification_blocking_cause().unwrap();
17893 assert!(cause.contains(&head));
17894 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17895 runner.state.event("land", "rebased successfully");
17897 let notice = crate::notices::run_ended(&runner.state).unwrap();
17898 assert!(!format!("{notice:?}").contains("rebased successfully"));
17899 assert!(runner.state.merge.is_none());
17900 }
17901 let before = std::fs::read_to_string(&phases).unwrap();
17902 runner.state.save().unwrap();
17903 runner.state = RunState::load(&runner.state.id).unwrap();
17904 runner.review_loop().await.unwrap();
17905 runner.gate().await.unwrap();
17906 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17907 assert_eq!(runner.state.verification_history.len(), 1);
17908 let history = &runner.state.verification_history[0];
17909 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17910 assert!(history.archived_at.is_some());
17911 assert_eq!(history.fixes_spent, Some(1));
17912 }
17913 }
17914
17915 #[cfg(unix)]
17916 #[tokio::test]
17917 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17918 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17919 let marker = tmp.path().join("gate-ran");
17920 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17921 runner.review_loop().await.unwrap();
17922 runner.gate().await.unwrap();
17923 assert_eq!(runner.state.status, RunStatus::Blocked);
17924 assert!(!marker.exists());
17925 assert!(runner.state.verification_history.is_empty());
17926 runner.state.gate_head = None;
17927 runner.gate().await.unwrap();
17928 assert!(marker.exists());
17929 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17930 assert_eq!(runner.state.verification_history.len(), 1);
17931 assert_eq!(runner.state.gate_fixes.len(), 1);
17932 }
17933
17934 #[cfg(unix)]
17935 #[tokio::test]
17936 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17937 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17938 runner.sync_to_base().await.unwrap();
17939 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17940 runner.state.reconcile_verification_head(&head);
17941 for outcomes in [
17943 Vec::new(),
17944 vec![CommandOutcome {
17945 command: "true".into(),
17946 code: None,
17947 output_tail: "cache unavailable".into(),
17948 duration_ms: 0,
17949 resource_blocked: true,
17950 }],
17951 ] {
17952 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17953 head: head.clone(),
17954 passed: false,
17955 outcomes,
17956 });
17957 runner.state.gate_ran = false;
17958 runner.state.gate.clear();
17959 runner.state.save().unwrap();
17960 runner.state = RunState::load(&runner.state.id).unwrap();
17961 runner.merge().await.unwrap();
17962 assert!(runner.state.merge.is_none());
17963 runner.review_loop().await.unwrap();
17964 runner.gate().await.unwrap();
17965 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17966 assert!(runner.state.gate_allows_progress());
17967 assert_eq!(runner.state.gate_fixes.len(), 1);
17968 assert!(runner.state.merge.is_none());
17969 }
17970 git::git(
17972 &runner.state.repo,
17973 &["commit", "--allow-empty", "-m", "external update"],
17974 )
17975 .await
17976 .unwrap();
17977 runner.merge().await.unwrap();
17978 assert!(runner.state.merge.is_none());
17979 assert_eq!(runner.state.status, RunStatus::Blocked);
17980 }
17981
17982 #[cfg(unix)]
17983 #[tokio::test]
17984 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17985 for waiting_on_gate in [false, true] {
17986 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17987 runner.sync_to_base().await.unwrap();
17988 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17989 runner.state.reconcile_verification_head(&head);
17990 if waiting_on_gate {
17991 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17992 head: head.clone(),
17993 passed: true,
17994 outcomes: Vec::new(),
17995 });
17996 }
17997 let cache = tmp.path().join("cache");
17998 runner.state.config.verify.e2e =
17999 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18000 runner.state.config.verify.gate = vec!["true".into()];
18001 runner.state.config.graph.timeout_resource_wait = 1;
18002 let owner =
18003 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18004 let guard =
18005 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18006 crate::cache::AcquireOutcome::Acquired(g) => g,
18007 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18008 };
18009 runner.review_loop().await.unwrap();
18010 runner.gate().await.unwrap();
18011 assert!(!runner.state.gate_ran);
18012 assert_eq!(runner.state.gate_fixes.len(), 1);
18013 assert!(runner.state.resource_block.is_some());
18014 runner.merge().await.unwrap();
18015 assert!(runner.state.merge.is_none());
18016 drop(guard);
18017 runner.state = RunState::load(&runner.state.id).unwrap();
18018 runner.state.resource_block = None;
18020 runner.review_loop().await.unwrap();
18021 runner.gate().await.unwrap();
18022 assert!(runner.state.gate_allows_progress());
18023 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18024 assert_eq!(runner.state.gate_fixes.len(), 1);
18025 }
18026 }
18027
18028 #[cfg(unix)]
18029 #[tokio::test]
18030 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18031 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18032 runner.sync_to_base().await.unwrap();
18033 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18034 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18035 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18037 runner.review_loop().await.unwrap();
18038 runner.gate().await.unwrap();
18039 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18040 assert_ne!(before, after);
18041 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18042 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18043 assert!(!runner.state.gate_allows_progress());
18044 runner.merge().await.unwrap();
18045 assert!(runner.state.merge.is_none());
18046 assert!(
18047 runner
18048 .state
18049 .verification_history
18050 .iter()
18051 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18052 );
18053
18054 runner.state.reviews[0].clean = false;
18056 runner.state.reviews[0].answered = 0;
18057 runner.state.config.verify.e2e = vec!["true".into()];
18058 runner.review_loop().await.unwrap();
18059 runner.gate().await.unwrap();
18060 runner.merge().await.unwrap();
18061 assert_eq!(runner.state.status, RunStatus::Blocked);
18062 assert!(runner.state.merge.is_none());
18063 assert!(
18064 runner
18065 .state
18066 .verification_blocking_cause()
18067 .unwrap()
18068 .contains("review blocked")
18069 );
18070 }
18071
18072 #[cfg(unix)]
18073 #[tokio::test]
18074 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18075 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18076 runner.state.gate_head = None;
18077 runner.state.config.verify.gate =
18078 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18079 runner.review_loop().await.unwrap();
18080 runner.gate().await.unwrap();
18081 assert_ne!(
18082 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18083 old
18084 );
18085 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18086 assert!(!runner.state.gate_allows_progress());
18087 assert_eq!(runner.state.status, RunStatus::Blocked);
18088 runner.merge().await.unwrap();
18089 assert!(runner.state.merge.is_none());
18090 }
18091
18092 #[cfg(unix)]
18093 #[tokio::test]
18094 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18095 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18096 let mut old_review = review_round(true, 0, 1, 1, false, true);
18097 old_review.head = head.clone();
18098 old_review.verified_head = Some(head.clone());
18099 runner.state.reviews = vec![old_review];
18100 runner.state.gate = vec![CommandOutcome {
18101 command: "old gate".into(),
18102 code: None,
18103 output_tail: "timed out after 2400s".into(),
18104 duration_ms: 2_400_000,
18105 resource_blocked: false,
18106 }];
18107 runner.state.gate_ran = true;
18108 runner.state.gate_head = Some(head.clone());
18109 runner.state.land_approval = Some(crate::run::LandApproval {
18110 question: "old-head-approval".into(),
18111 head: head.clone(),
18112 });
18113 runner.state.authorize_base_sync(&task).unwrap();
18114 runner.sync_to_base().await.unwrap();
18115 assert!(!runner.state.base_sync_blocked());
18116 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18117 assert_ne!(head, fresh);
18118 assert_eq!(runner.state.reviews.len(), 1);
18119 assert!(!runner.state.gate_ran);
18120 let saved = &runner.state.base_sync_recovery.authorizations[0];
18121 assert_eq!(saved.reviews[0].head, head);
18122 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18123 assert!(saved.gate_ran);
18124 assert!(!saved.gate[0].ok());
18125 runner.review_loop().await.unwrap();
18126 runner.gate().await.unwrap();
18127 assert_eq!(runner.state.reviews[0].head, head);
18128 assert_eq!(
18129 runner.state.reviews[0].verified_head.as_deref(),
18130 Some(head.as_str())
18131 );
18132 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18133 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18134 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18135 assert!(runner.state.gate_ran);
18136 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18137 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18138 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18139 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18140 assert_eq!(
18141 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18142 "retained change"
18143 );
18144 }
18145
18146 #[cfg(unix)]
18147 #[tokio::test]
18148 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18149 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18150 let queue = crate::queue::Queue::open();
18151 let mut owner = queue.get(&task).unwrap();
18152 owner.release_operator("test-resume", None);
18153 queue.put(&mut owner).unwrap();
18154 let generation = owner.resumption.as_ref().unwrap().generation;
18155 runner.state.task_instruction = Some(owner.instruction.clone());
18156 runner.state.driver_exited = true;
18157 runner.state.save().unwrap();
18158 let home = crate::run::home();
18159 let _claim = queue.claim(&task).unwrap();
18160 owner.resumption.as_mut().unwrap().at =
18161 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18162 queue.put(&mut owner).unwrap();
18163 assert!(
18164 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18165 .await
18166 .is_err(),
18167 "an approval from before the stop grants nothing"
18168 );
18169 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18170 queue.put(&mut owner).unwrap();
18171 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18172 .await
18173 .unwrap();
18174 assert!(
18175 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18176 .await
18177 .is_err()
18178 );
18179 runner.state = RunState::load(&runner.state.id).unwrap();
18180 assert_eq!(
18181 runner.state.base_sync_recovery.authorizations[0].generation,
18182 Some(generation)
18183 );
18184 runner.sync_to_base().await.unwrap();
18185 let repo = runner.state.repo.clone();
18186 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18187 git::git(&repo, &["checkout", "main"]).await.unwrap();
18188 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18189 git::git(&repo, &["add", "."]).await.unwrap();
18190 git::git(&repo, &["commit", "-m", "move base again"])
18191 .await
18192 .unwrap();
18193 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18194 git::git(&repo, &["checkout", "fix/retained"])
18195 .await
18196 .unwrap();
18197 runner.sync_to_base().await.unwrap();
18198 assert!(!runner.state.base_sync_blocked());
18199 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18200 assert_eq!(runner.state.base_episode_used(), 2);
18201 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18202 assert_ne!(fresh, head);
18203 assert!(!runner.state.base_recovery_pending());
18204 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18206 queue.put(&mut owner).unwrap();
18207 assert!(
18208 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18209 .await
18210 .is_err()
18211 );
18212 runner.state = RunState::load(&runner.state.id).unwrap();
18213 runner.sync_to_base().await.unwrap();
18214 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18215 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18216 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18217 }
18218
18219 #[cfg(unix)]
18220 #[tokio::test]
18221 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18222 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18223 runner.state.base_sync = None;
18224 runner.state.status = RunStatus::Blocked;
18225 let mut second = runner.state.candidates[0].clone();
18226 second.label = 'B';
18227 runner.state.candidates.push(second);
18228 assert!(runner.state.votes.is_empty());
18229 assert!(runner.state.deliberation.is_empty());
18230 runner.deliberate().await.unwrap();
18231 runner.vote().await.unwrap();
18232 assert_eq!(runner.state.status, RunStatus::Blocked);
18233 assert!(runner.state.votes.is_empty());
18234 assert!(runner.state.deliberation.is_empty());
18235 }
18236
18237 fn runner_at(status: RunStatus) -> Runner {
18240 let mut state = RunState::new(
18241 PathBuf::from("/nonexistent/repo"),
18242 "main".to_owned(),
18243 "deadbeef".to_owned(),
18244 "task".to_owned(),
18245 Config::default(),
18246 );
18247 state.config.disk.min_free_bytes = 0;
18249 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18252 state.id = format!(
18253 "20000101-000000-{:04x}",
18254 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18255 );
18256 state.status = status;
18257 Runner {
18258 state,
18259 roles: ResolvedRoles {
18260 implementers: Vec::new(),
18261 judges: Vec::new(),
18262 reviewers: Vec::new(),
18263 fixer: None,
18264 conductor: conductor(),
18265 implementer_roster: Vec::new(),
18266 judge_roster: Vec::new(),
18267 reviewer_roster: Vec::new(),
18268 },
18269 sem: Arc::new(Semaphore::new(1)),
18270 pause: Pause::new(),
18271 interrupt: Pause::new(),
18272 }
18273 }
18274
18275 #[tokio::test]
18276 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18277 crate::run::pin_test_home();
18278 let temp = tempfile::tempdir().unwrap();
18279 let root = temp.path().canonicalize().unwrap();
18280 let repo = root.join("repo");
18281 std::fs::create_dir(&repo).unwrap();
18282 init_repo(&repo);
18283 git::git(&repo, &["config", "core.autocrlf", "false"])
18284 .await
18285 .unwrap();
18286 let mut runner = runner_at(RunStatus::Blocked);
18287 runner.state.repo = repo.clone();
18288 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18289 runner.state.config.agents = vec![conductor()];
18290 runner.state.config.graph.selected_implementer = Some("conductor".into());
18291 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18292 let (shell, command, setup_output) = if cfg!(windows) {
18293 (
18294 vec!["cmd".into(), "/C".into()],
18295 "echo ready>seat-ready",
18296 "ready\r\n",
18297 )
18298 } else {
18299 (
18300 vec!["sh".into(), "-c".into()],
18301 "printf ready > seat-ready",
18302 "ready",
18303 )
18304 };
18305 runner.state.config.verify.shell = Some(shell);
18306 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18307 run: Some(command.into()),
18308 ..Default::default()
18309 }];
18310 runner.state.save().unwrap();
18311 let path = runner.state.worktree_root().join("judge-1");
18312 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18313 .await
18314 .unwrap();
18315 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18316 git::git(&path, &["add", "."]).await.unwrap();
18317 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18318 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18319 .await
18320 .unwrap();
18321 git::git(&path, &["checkout", "--detach", &head])
18322 .await
18323 .unwrap();
18324 let evidence = crate::fold_evidence::snapshot(
18325 &runner.state,
18326 &crate::run::home().canonicalize().unwrap(),
18327 &head,
18328 64 * 1024 * 1024,
18329 )
18330 .await
18331 .unwrap();
18332 let mut contents = BTreeMap::new();
18333 contents.insert(
18334 repo.join("logs/HEAD"),
18335 format!("{head} {head} fixture\n").into_bytes(),
18336 );
18337 let record = serde_json::json!({
18338 "version":1, "run":runner.state.id, "source":path, "head":head,
18339 "evidence":evidence, "admin":{}, "admin_contents":contents,
18340 "history":{head.clone():evidence}, "phase":"evicted"
18341 });
18342 std::fs::write(
18343 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18344 serde_json::to_vec(&record).unwrap(),
18345 )
18346 .unwrap();
18347 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18348 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18349 .await
18350 .unwrap();
18351 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18352 assert!(
18353 !git::git_raw(&repo, &["cat-file", "-e", &head])
18354 .await
18355 .unwrap()
18356 .ok()
18357 );
18358 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18359 let error = resumed.execute().await.unwrap_err();
18360 assert!(
18361 error
18362 .to_string()
18363 .contains("selected implementation is held"),
18364 "{error:#}"
18365 );
18366 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18367 assert_eq!(
18368 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18369 setup_output
18370 );
18371 assert_eq!(
18372 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18373 "unrelated seat"
18374 );
18375 let error = resumed.execute().await.unwrap_err();
18376 assert!(
18377 error
18378 .to_string()
18379 .contains("selected implementation is held"),
18380 "{error:#}"
18381 );
18382 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18383 git::git(&path, &["add", "."]).await.unwrap();
18384 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18385 let next = git::git(
18386 &path,
18387 &[
18388 "commit-tree",
18389 &next_tree,
18390 "-p",
18391 &head,
18392 "-m",
18393 "next review checkpoint",
18394 ],
18395 )
18396 .await
18397 .unwrap();
18398 git::git(&path, &["checkout", "--detach", &next])
18399 .await
18400 .unwrap();
18401 resumed.state.reviews.push(
18402 serde_json::from_value(serde_json::json!({
18403 "round":1, "head":next, "reviews":[]
18404 }))
18405 .unwrap(),
18406 );
18407 resumed.state.save().unwrap();
18408 let error = resumed.execute().await.unwrap_err();
18409 assert!(
18410 error
18411 .to_string()
18412 .contains("selected implementation is held"),
18413 "{error:#}"
18414 );
18415 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18416 }
18417
18418 #[tokio::test]
18419 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18420 crate::run::pin_test_home();
18421 let mut runner = runner_at(RunStatus::Gating);
18422 runner.state.save().unwrap();
18423 let id = runner.state.id.clone();
18424 let passes = runner.state.passes.len();
18425 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18426 let mut claiming = Box::pin(runner.claim_driver());
18427 std::future::poll_fn(|cx| {
18428 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18429 std::task::Poll::Ready(())
18430 })
18431 .await;
18432 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18433 drop(owner);
18434 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18435 .await
18436 .unwrap()
18437 .unwrap();
18438 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18439 assert!(execution.runner.state.resource_block.is_none());
18440 assert_eq!(
18441 execution.runner.state.liveness(false),
18442 crate::run::Liveness::Live
18443 );
18444 drop(execution);
18445 assert!(RunState::load(&id).unwrap().driver_exited);
18446 }
18447
18448 #[tokio::test]
18449 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18450 crate::run::pin_test_home();
18451 let mut runner = runner_at(RunStatus::Gating);
18452 runner.state.save().unwrap();
18453 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18456 assert!(runner.claim_driver().await.is_err());
18457 assert!(runner.state.driver_exited);
18458 assert_eq!(runner.state.driver_pid, None);
18459 let disk = RunState::load(&runner.state.id).unwrap();
18460 assert!(disk.driver_exited);
18461 assert_eq!(disk.driver_pid, None);
18462 assert_eq!(disk.status, RunStatus::Gating);
18463 }
18464
18465 #[tokio::test]
18466 async fn an_early_graph_error_releases_its_driver() {
18467 crate::run::pin_test_home();
18468 let mut runner = runner_at(RunStatus::Blocked);
18469 runner.state.config.graph.selected_implementer = Some("selected".into());
18470 runner.state.save().unwrap();
18471 assert!(runner.execute().await.is_err());
18472 let disk = RunState::load(&runner.state.id).unwrap();
18473 assert!(disk.driver_exited);
18474 assert_eq!(disk.driver_pid, None);
18475 assert_eq!(disk.status, RunStatus::Blocked);
18476 }
18477
18478 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18479 crate::run::MergeRepair {
18480 question: "repair-question".into(),
18481 head: "old-head".into(),
18482 pr: "https://github.com/o/r/pull/42".into(),
18483 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18484 gate: vec![outcome(Some(0), false)],
18485 gate_head: Some("old-head".into()),
18486 gate_fixes: Vec::new(),
18487 pre_gate: Vec::new(),
18488 phase,
18489 failure: None,
18490 }
18491 }
18492
18493 #[tokio::test]
18494 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18495 crate::run::pin_test_home();
18496 let mut runner = runner_at(RunStatus::Landing);
18497 runner.state.merge_repairs.push(merge_repair_request(
18498 crate::run::MergeRepairPhase::Requested,
18499 ));
18500 runner.state.save().unwrap();
18501 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18502 assert!(
18503 runner
18504 .execute()
18505 .await
18506 .unwrap_err()
18507 .to_string()
18508 .contains("already running")
18509 );
18510 assert_eq!(
18511 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18512 crate::run::MergeRepairPhase::Requested
18513 );
18514 drop(claim);
18515 let mut disk = runner.state.clone();
18516 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18517 disk.save().unwrap();
18518 assert!(
18519 runner
18520 .execute()
18521 .await
18522 .unwrap_err()
18523 .to_string()
18524 .contains("decision changed")
18525 );
18526 assert_eq!(
18527 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18528 crate::run::MergeRepairPhase::Complete
18529 );
18530 }
18531
18532 #[test]
18533 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18534 crate::run::pin_test_home();
18535 let mut runner = runner_at(RunStatus::Landing);
18536 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18537 runner.state.reviews = request.reviews.clone();
18538 runner.state.gate = request.gate.clone();
18539 runner.state.gate_ran = true;
18540 runner.state.land_approval = Some(crate::run::LandApproval {
18541 question: "old".into(),
18542 head: "old-head".into(),
18543 });
18544 runner.state.merge_repairs.push(request);
18545 runner.begin_merge_repair_review(0).unwrap();
18546 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18547 assert!(!runner.state.gate_status().ok());
18548 assert!(runner.state.land_approval.is_none());
18549 let disk = RunState::load(&runner.state.id).unwrap();
18550 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18551 assert!(disk.merge_repairs[0].gate[0].ok());
18552 assert_eq!(
18553 disk.merge_repairs[0].phase,
18554 crate::run::MergeRepairPhase::Reviewing
18555 );
18556 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18557 assert!(disk.reviews.is_empty());
18558 }
18559
18560 #[tokio::test]
18561 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18562 crate::run::pin_test_home();
18563 for phase in [
18564 crate::run::MergeRepairPhase::Fixing,
18565 crate::run::MergeRepairPhase::Requested,
18566 ] {
18567 let mut runner = runner_at(RunStatus::Landing);
18568 runner.state.config.graph.land_rounds = 0;
18569 runner.state.merge_repairs.push(merge_repair_request(phase));
18570 runner.state.save().unwrap();
18571 assert!(!runner.repair_merge_review().await.unwrap());
18572 assert_eq!(runner.state.status, RunStatus::Blocked);
18573 assert_eq!(
18574 runner.state.merge_repairs[0].phase,
18575 crate::run::MergeRepairPhase::Failed
18576 );
18577 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18578 assert!(
18579 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18580 "interrupted"
18581 } else {
18582 "budget exhausted"
18583 })
18584 );
18585 assert!(runner.state.merge.is_none());
18586 }
18587 }
18588
18589 #[test]
18593 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18594 crate::run::pin_test_home();
18595 let mut runner = runner_at(RunStatus::Implementing);
18596 let interrupt = Pause::new();
18597 runner.watch_interrupt(interrupt.clone());
18598
18599 interrupt.park_because("task a1b2 asked to run first");
18600
18601 assert!(runner.park_here().expect("park_here"));
18602 assert!(runner.state.parked);
18603 let last = runner.state.events.last().expect("a park event");
18604 assert_eq!(last.node, "park");
18605 assert!(
18606 last.message.contains("task a1b2 asked to run first"),
18607 "expected the interrupt reason in {:?}",
18608 last.message
18609 );
18610 }
18611
18612 #[test]
18615 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18616 crate::run::pin_test_home();
18617 let mut runner = runner_at(RunStatus::Implementing);
18618 runner.state.begin_pass();
18619 let shutdown = Pause::new();
18620 runner.on_pause(shutdown.clone());
18621 let upgrade = ParkCause::Upgrade {
18622 from: "0.121.0".to_owned(),
18623 to: Some("0.121.1".to_owned()),
18624 };
18625 shutdown.park_for(upgrade.clone());
18626 shutdown.park_for(ParkCause::Operator);
18627
18628 assert!(runner.park_here().expect("park_here"));
18629 assert_eq!(
18630 runner.state.passes[0].end,
18631 Some(crate::run::PassEnd::Parked { cause: upgrade })
18632 );
18633 let last = runner.state.events.last().expect("a park event");
18634 assert!(last.message.contains("0.121.0"), "{}", last.message);
18635 }
18636
18637 #[tokio::test]
18638 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18639 crate::run::pin_test_home();
18640 let mut runner = runner_at(RunStatus::Reviewing);
18641 runner.state.save().unwrap();
18642 let mut execution = runner.claim_driver().await.expect("first execution");
18643 assert_eq!(execution.runner.state.passes.len(), 1);
18644 execution
18645 .driver
18646 .finish(&mut execution.runner.state)
18647 .await
18648 .unwrap();
18649 }
18650
18651 #[tokio::test]
18652 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18653 crate::run::pin_test_home();
18654 let mut runner = runner_at(RunStatus::Reviewing);
18655 runner.state.config.agents = vec![conductor()];
18656 runner.state.config.graph.timeout_resource_wait = 0;
18657 runner.state.save().unwrap();
18658 let id = runner.state.id.clone();
18659 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18660 assert!(runner.claim_driver().await.is_err());
18661 drop(owner);
18662 drop(runner);
18663 let mut resumed = Runner::resume(&id).unwrap();
18664 assert_eq!(resumed.state.driver_claimed, Some(false));
18665 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18666 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18667 let mut execution = resumed
18668 .claim_driver()
18669 .await
18670 .expect("retry initial claim after reload");
18671 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18672 assert_eq!(execution.runner.state.passes.len(), 1);
18673 execution
18674 .driver
18675 .finish(&mut execution.runner.state)
18676 .await
18677 .unwrap();
18678 }
18679
18680 #[tokio::test]
18681 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18682 crate::run::pin_test_home();
18683 let mut runner = runner_at(RunStatus::Reviewing);
18684 runner.state.config.agents = vec![conductor()];
18685 runner.state.save().unwrap();
18686 let id = runner.state.id.clone();
18687 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18688 let mut resumed = Runner::resume(&id).unwrap();
18689 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18690 assert!(resumed.claim_driver().await.is_err());
18691 assert!(RunState::load(&id).unwrap().passes.is_empty());
18692 drop(lease);
18693 let mut execution = resumed.claim_driver().await.unwrap();
18694 execution
18695 .driver
18696 .finish(&mut execution.runner.state)
18697 .await
18698 .unwrap();
18699 }
18700
18701 #[tokio::test]
18702 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18703 crate::run::pin_test_home();
18704 let mut runner = runner_at(RunStatus::Reviewing);
18705 let state = &mut runner.state;
18706 state.config.agents = vec![conductor()];
18707 state.driver_claimed = None;
18708 state.driver_pid = Some(std::process::id());
18709 state.driver_started_at = None;
18710 state.begin_pass();
18711 state.save().unwrap();
18712 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18713 std::fs::write(
18714 state.dir().join("driver.json"),
18715 serde_json::to_vec(&serde_json::json!({
18716 "pid": state.driver_pid, "started_at": null,
18717 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18718 }))
18719 .unwrap(),
18720 )
18721 .unwrap();
18722 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18723 drop(lease);
18724 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18725 let mut resumed = Runner::resume(&state.id).unwrap();
18726 assert!(!resumed.state.driver_exited);
18727 let mut execution = resumed
18728 .claim_driver()
18729 .await
18730 .expect("lease proves previous execution ended");
18731 assert_eq!(execution.runner.state.passes.len(), 2);
18732 execution
18733 .driver
18734 .finish(&mut execution.runner.state)
18735 .await
18736 .unwrap();
18737 }
18738
18739 #[tokio::test]
18740 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18741 crate::run::pin_test_home();
18742 let mut runner = runner_at(RunStatus::Reviewing);
18743 let mut json = serde_json::to_value(&runner.state).unwrap();
18744 json.as_object_mut().unwrap().remove("driver_claimed");
18745 runner.state = serde_json::from_value(json).unwrap();
18746 assert_eq!(runner.state.driver_claimed, None);
18747 runner.state.save().unwrap();
18748 let mut execution = runner
18749 .claim_driver()
18750 .await
18751 .expect("the execution lease, not a missing marker, excludes a second driver");
18752 assert_eq!(execution.runner.state.passes.len(), 1);
18753 execution
18754 .driver
18755 .finish(&mut execution.runner.state)
18756 .await
18757 .unwrap();
18758 }
18759
18760 #[tokio::test]
18761 async fn driver_admission_refuses_a_proven_live_driver() {
18762 crate::run::pin_test_home();
18763 let mut runner = runner_at(RunStatus::Reviewing);
18764 let pid = std::process::id();
18765 runner.state.driver_pid = Some(pid);
18766 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18767 assert!(runner.state.driver_started_at.is_some());
18768 runner.state.save().unwrap();
18769 let error = runner
18770 .claim_driver()
18771 .await
18772 .err()
18773 .expect("a live driver must refuse");
18774 assert!(error.to_string().contains("live driver"), "{error:#}");
18775 let saved = RunState::load(&runner.state.id).unwrap();
18776 assert_eq!(saved.driver_pid, Some(pid));
18777 assert_eq!(saved.status, RunStatus::Reviewing);
18778 assert!(saved.passes.is_empty());
18779 }
18780
18781 #[test]
18782 fn parking_resource_halt_persists_the_pass_once() {
18783 crate::run::pin_test_home();
18784 let mut runner = runner_at(RunStatus::Reviewing);
18785 runner.state.begin_pass();
18786 runner.state.resource_block = Some(crate::run::ResourceBlock {
18787 node: "fix".into(),
18788 kind: "shared build cache".into(),
18789 detail: "cancelled".into(),
18790 at: Timestamp::now(),
18791 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18792 });
18793 let cause = ParkCause::Upgrade {
18794 from: "old".into(),
18795 to: Some("new".into()),
18796 };
18797 runner.pause.park_for(cause.clone());
18798 runner.stop_for_resource().unwrap();
18801 assert!(runner.resource_halt().unwrap());
18802 assert!(runner.park_here().unwrap());
18803 let saved = RunState::load(&runner.state.id).unwrap();
18804 assert!(saved.parked);
18805 assert_eq!(saved.status, RunStatus::Reviewing);
18806 assert!(saved.passes[0].ended_at.is_some());
18807 assert_eq!(
18808 saved.passes[0].end,
18809 Some(crate::run::PassEnd::Parked {
18810 cause: cause.clone()
18811 })
18812 );
18813 assert_eq!(
18814 saved.resource_block.unwrap().cause,
18815 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18816 );
18817 assert_eq!(
18818 saved
18819 .events
18820 .iter()
18821 .filter(|event| event.node == "park")
18822 .count(),
18823 1
18824 );
18825 }
18826
18827 #[test]
18835 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18836 crate::run::pin_test_home();
18837 let mut runner = runner_at(RunStatus::Implementing);
18838 let shutdown = Pause::new();
18839 runner.on_pause(shutdown.clone());
18840 let interrupt = Pause::new();
18841 runner.watch_interrupt(interrupt.clone());
18842
18843 assert!(!runner.park_here().expect("park_here"));
18845 assert!(!runner.state.parked);
18846
18847 interrupt.park_because("test");
18849 assert!(!shutdown.parked());
18850 assert!(runner.park_here().expect("park_here"));
18851 }
18852
18853 #[tokio::test]
18867 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18868 crate::run::pin_test_home();
18869 let mut runner = runner_at(RunStatus::Implementing);
18870 let interrupt = Pause::new();
18871 runner.watch_interrupt(interrupt.clone());
18872
18873 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18874 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18875
18876 let node = async move {
18880 started_tx.send(()).expect("send started");
18881 finish_rx.await.expect("recv finish");
18882 "node finished"
18883 };
18884
18885 let interrupter = async move {
18886 started_rx.await.expect("recv started");
18887 interrupt.park_because("higher-priority task waiting");
18889 tokio::task::yield_now().await;
18893 finish_tx.send(()).expect("send finish");
18894 };
18895
18896 let (node_result, ()) = tokio::join!(node, interrupter);
18897 assert_eq!(
18898 node_result, "node finished",
18899 "the in-flight call ran to completion"
18900 );
18901
18902 assert!(runner.park_here().expect("park_here"));
18905 assert!(runner.state.parked);
18906 }
18907
18908 #[test]
18914 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18915 crate::run::pin_test_home();
18916 let mut runner = runner_at(RunStatus::Judging);
18917 runner.state.config.agents = vec![conductor()];
18921 runner.state.candidates = vec![Candidate {
18922 index: 0,
18923 label: 'A',
18924 agent: "alpha".to_owned(),
18925 branch: "magi/x/A".to_owned(),
18926 worktree: PathBuf::from("/nonexistent/worktree"),
18927 summary: "did the thing".to_owned(),
18928 stat: "1 file changed".to_owned(),
18929 files: 1,
18930 commits: 1,
18931 empty: false,
18932 failed: None,
18933 verified_noop: None,
18934 duration_ms: 1234,
18935 folded: false,
18936 }];
18937 let run_id = runner.state.id.clone();
18938
18939 let interrupt = Pause::new();
18940 runner.watch_interrupt(interrupt.clone());
18941 interrupt.park_because("task c3d4 asked to run first");
18942 assert!(runner.park_here().expect("park_here"));
18943
18944 let resumed = Runner::resume(&run_id).expect("resume");
18945 assert_eq!(resumed.state.candidates.len(), 1);
18946 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18947 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18948 assert_eq!(resumed.state.status, runner.state.status);
18949 assert!(
18950 resumed.state.parked,
18951 "still parked until `execute` actually walks the graph again"
18952 );
18953 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18954 }
18955
18956 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18958 let mut q = ask::Question::new(
18959 run.to_owned(),
18960 "implement".to_owned(),
18961 "impl-A".to_owned(),
18962 "Which storage backend should the cache use?".to_owned(),
18963 String::new(),
18964 vec!["SQLite".to_owned(), "Redis".to_owned()],
18965 );
18966 store.put(&mut q).unwrap();
18967 q
18968 }
18969
18970 #[test]
18971 fn a_failed_runs_open_question_is_abandoned() {
18972 ask_test_home();
18973 let store = ask::Questions::open();
18974 let mut runner = runner_at(RunStatus::Failed);
18975 let run = runner.state.id.clone();
18976 let q = ask_open_question(&store, &run);
18977
18978 runner.settle_questions();
18979
18980 let back = store.get(&q.id).unwrap();
18981 assert!(
18982 !back.status.open(),
18983 "the seat that asked died with the run; nobody is left to read an answer"
18984 );
18985 assert!(
18986 back.detail.contains(&run) && back.detail.contains("failed"),
18987 "the reason names what the run became, not just that it is gone: {}",
18988 back.detail
18989 );
18990 }
18991
18992 #[test]
18993 fn a_merged_runs_open_question_is_abandoned_too() {
18994 ask_test_home();
18995 let store = ask::Questions::open();
18996 for status in [RunStatus::Merged, RunStatus::Ready] {
18999 let mut runner = runner_at(status);
19000 let run = runner.state.id.clone();
19001 let q = ask_open_question(&store, &run);
19002
19003 runner.settle_questions();
19004
19005 let back = store.get(&q.id).unwrap();
19006 assert!(
19007 !back.status.open(),
19008 "{status:?} run's question must not outlive the run"
19009 );
19010 }
19011 }
19012
19013 #[test]
19014 fn a_still_resumable_runs_open_question_is_left_alone() {
19015 ask_test_home();
19016 let store = ask::Questions::open();
19017 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19023 let mut runner = runner_at(status);
19024 let run = runner.state.id.clone();
19025 let q = ask_open_question(&store, &run);
19026
19027 runner.settle_questions();
19028
19029 let back = store.get(&q.id).unwrap();
19030 assert!(
19031 back.status.open(),
19032 "{status:?} is still alive; the question must still be waiting"
19033 );
19034 }
19035 }
19036
19037 #[test]
19038 fn settle_questions_never_touches_an_already_answered_question() {
19039 ask_test_home();
19040 let store = ask::Questions::open();
19041 let mut runner = runner_at(RunStatus::Failed);
19042 let run = runner.state.id.clone();
19043 let mut q = ask_open_question(&store, &run);
19044 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19045 .unwrap();
19046 store.put(&mut q).unwrap();
19047
19048 runner.settle_questions();
19053 runner.settle_questions();
19054
19055 let back = store.get(&q.id).unwrap();
19056 assert_eq!(
19057 back.status,
19058 ask::QuestionStatus::Answered,
19059 "a real answer is a decision on record, never overwritten by a sweep"
19060 );
19061 }
19062
19063 #[tokio::test]
19074 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19075 crate::run::pin_test_home();
19076 let tmp = tempfile::tempdir().expect("tempdir");
19077 let repo = tmp.path().join("repo");
19078 std::fs::create_dir_all(&repo).unwrap();
19079 init_repo(&repo);
19080
19081 let mut config = Config::default();
19082 config.graph.worktree_root = Some(tmp.path().join("wt"));
19083
19084 let mut state = RunState::new(
19085 repo.clone(),
19086 "main".to_owned(),
19087 "deadbeef".to_owned(),
19088 "task".to_owned(),
19089 config,
19090 );
19091 let root = state.worktree_root();
19092 let wt_a = root.join("cand-A");
19093 let wt_b = root.join("cand-B");
19094 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19095 .await
19096 .expect("worktree A");
19097 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19098 .await
19099 .expect("worktree B");
19100
19101 state.candidates = vec![
19102 Candidate {
19103 index: 0,
19104 label: 'A',
19105 agent: "alpha".to_owned(),
19106 branch: "magi/x/A".to_owned(),
19107 worktree: wt_a.clone(),
19108 summary: String::new(),
19109 stat: String::new(),
19110 files: 0,
19111 commits: 0,
19112 empty: false,
19113 failed: None,
19114 verified_noop: None,
19115 duration_ms: 0,
19116 folded: false,
19117 },
19118 Candidate {
19119 index: 1,
19120 label: 'B',
19121 agent: "beta".to_owned(),
19122 branch: "magi/x/B".to_owned(),
19123 worktree: wt_b.clone(),
19124 summary: String::new(),
19125 stat: String::new(),
19126 files: 0,
19127 commits: 0,
19128 empty: false,
19129 failed: None,
19130 verified_noop: None,
19131 duration_ms: 0,
19132 folded: false,
19133 },
19134 ];
19135 state.tally = Some(Tally {
19136 first_choice: BTreeMap::from([('A', 1)]),
19137 borda: BTreeMap::new(),
19138 winner: 'A',
19139 rankings: 1,
19140 unanimous_initial: true,
19141 deliberated: false,
19142 changed_votes: 0,
19143 unanimous_final: true,
19144 tie_break: None,
19145 judges: 1,
19146 present: 1,
19147 quorum: 1,
19148 met_quorum: true,
19149 uncontested: None,
19150 });
19151 state.status = RunStatus::Ready;
19152
19153 fold_run(&mut state, false, &crate::run::home())
19154 .await
19155 .expect("fold_run");
19156
19157 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19158 assert!(
19159 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19160 "the unmerged winner's branch survives"
19161 );
19162 assert!(
19163 !state.candidates[0].folded,
19164 "the winner is not marked folded"
19165 );
19166
19167 assert!(!wt_b.exists(), "the loser's worktree is removed");
19168 assert!(
19169 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19170 "the loser's branch is removed"
19171 );
19172 assert!(state.candidates[1].folded, "the loser is marked folded");
19173 }
19174
19175 #[tokio::test]
19176 #[cfg(unix)]
19177 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19178 use std::os::fd::AsRawFd as _;
19179 use std::time::Duration;
19180
19181 let tmp = tempfile::tempdir().unwrap();
19182 let repo = tmp.path().join("repo");
19183 std::fs::create_dir_all(&repo).unwrap();
19184 init_repo(&repo);
19185 let common = repo.join(".git");
19186 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19187 let home = tmp.path().join("home");
19188 let mut config = Config::default();
19189 config.coordination = crate::config::test_coordination();
19190 config.graph.worktree_root = Some(tmp.path().join("wt"));
19191 config.graph.timeout_resource_wait = 1;
19192 let mut state = RunState::new(
19193 repo.clone(),
19194 "main".into(),
19195 "deadbeef".into(),
19196 "task".into(),
19197 config,
19198 );
19199 let root = state.worktree_root();
19200 let candidate = root.join("cand-A");
19201 let reviewer = root.join("review-1");
19202 let branch = state.branch_for('A');
19203 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19204 .await
19205 .unwrap();
19206 git::worktree_add_detached(&repo, &reviewer, "main")
19207 .await
19208 .unwrap();
19209 git::acquire_worktree_config(&repo).await.unwrap();
19210 state.enabled_worktree_config = true;
19211 state.status = RunStatus::Ready;
19212 state.candidates.push(Candidate {
19213 index: 0,
19214 label: 'A',
19215 agent: "implement".into(),
19216 branch: branch.clone(),
19217 worktree: candidate.clone(),
19218 summary: String::new(),
19219 stat: String::new(),
19220 files: 0,
19221 commits: 0,
19222 empty: false,
19223 failed: None,
19224 verified_noop: None,
19225 duration_ms: 0,
19226 folded: false,
19227 });
19228 state.save_under(&home).unwrap();
19229 let config_before = std::fs::read(common.join("config")).unwrap();
19230 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19231 .await
19232 .unwrap();
19233 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19236 assert_eq!(
19237 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19238 0
19239 );
19240 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19241 assert!(
19242 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19243 .await
19244 .is_err()
19245 );
19246 assert!(candidate.exists());
19247 assert!(reviewer.exists());
19248 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19249 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19250 assert_eq!(
19251 git::git(&repo, &["worktree", "list", "--porcelain"])
19252 .await
19253 .unwrap(),
19254 worktrees_before
19255 );
19256 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19257 .await
19258 .unwrap()
19259 .unwrap_err();
19260 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19261 drop(fold);
19262 assert!(candidate.exists() && reviewer.exists());
19263 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19264 assert_eq!(
19265 state.resource_block.as_ref().unwrap().kind,
19266 "repository coordination"
19267 );
19268 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19269 drop(external);
19270 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19271 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19272 .await
19273 .unwrap()
19274 .unwrap();
19275 drop(fold);
19276 assert!(removed.contains(&branch));
19277 assert!(!root.exists());
19278 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19279 assert!(state.candidates[0].folded);
19280 assert!(!state.enabled_worktree_config);
19281 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19282 let enabled = git::git_raw(
19283 &repo,
19284 &["config", "--local", "--get", "extensions.worktreeConfig"],
19285 )
19286 .await
19287 .unwrap();
19288 assert!(!enabled.ok(), "the last config reference was released");
19289 }
19290
19291 #[tokio::test]
19294 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19295 let tmp = tempfile::tempdir().expect("tempdir");
19296 let repo = tmp.path().join("repo");
19297 std::fs::create_dir_all(&repo).unwrap();
19298 init_repo(&repo);
19299 let home = tmp.path().join("home");
19300
19301 let mut config = Config::default();
19302 config.graph.worktree_root = Some(tmp.path().join("wt"));
19303 let mut state = RunState::new(
19304 repo.clone(),
19305 "main".to_owned(),
19306 "deadbeef".to_owned(),
19307 "task".to_owned(),
19308 config,
19309 );
19310 git::git(&repo, &["branch", "magi/x/A", "main"])
19312 .await
19313 .expect("branch");
19314 state.candidates = vec![Candidate {
19315 index: 0,
19316 label: 'A',
19317 agent: "alpha".to_owned(),
19318 branch: "magi/x/A".to_owned(),
19319 worktree: state.worktree_root().join("cand-A"),
19320 summary: String::new(),
19321 stat: String::new(),
19322 files: 0,
19323 commits: 0,
19324 empty: false,
19325 failed: None,
19326 verified_noop: None,
19327 duration_ms: 0,
19328 folded: true,
19329 }];
19330 state.released_to = Some("20260901-000000-new1".to_owned());
19331 state.released_branches = vec!["magi/x/A".to_owned()];
19332
19333 fold_run(&mut state, true, &home).await.expect("fold_run");
19334 state.status = RunStatus::Merged;
19335 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19336 state.merge = Some(crate::run::MergeOutcome {
19337 mode: MergeMode::Pr,
19338 ok: true,
19339 detail: "confirmed merged".into(),
19340 empty: false,
19341 });
19342 fold_run(&mut state, true, &home)
19343 .await
19344 .expect("merged fold");
19345
19346 assert!(
19347 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19348 "the handed-over branch survives a fold"
19349 );
19350 }
19351
19352 #[tokio::test]
19356 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19357 let tmp = tempfile::tempdir().expect("tempdir");
19358 let repo = tmp.path().join("repo");
19359 std::fs::create_dir_all(&repo).unwrap();
19360 init_repo(&repo);
19361 let run = |args: &[&str]| {
19362 let out = std::process::Command::new("git")
19363 .quiet()
19364 .args(args)
19365 .current_dir(&repo)
19366 .output()
19367 .expect("spawn git");
19368 assert!(out.status.success(), "git {args:?}");
19369 };
19370 run(&["branch", "magi/x/A"]);
19371 run(&["checkout", "-q", "-b", "magi/x/B"]);
19372 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19373 run(&["add", "-A"]);
19374 run(&["commit", "-q", "-m", "work"]);
19375 run(&["checkout", "-q", "main"]);
19376
19377 let probe = RunState::new(
19381 repo.clone(),
19382 "main".to_owned(),
19383 "deadbeef".to_owned(),
19384 "task".to_owned(),
19385 Config::default(),
19386 );
19387 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19388 let bare = tmp.path().join("origin.git");
19389 let out = std::process::Command::new("git")
19390 .quiet()
19391 .args(["init", "-q", "--bare"])
19392 .arg(&bare)
19393 .output()
19394 .expect("spawn git");
19395 assert!(out.status.success(), "git init --bare");
19396 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19397 run(&["push", "-q", "origin", "main"]);
19398
19399 let mut state = RunState::new(
19400 repo.clone(),
19401 "main".to_owned(),
19402 "deadbeef".to_owned(),
19403 "task".to_owned(),
19404 Config::default(),
19405 );
19406 state.seeds = vec![refs::Seed {
19407 token: "magi/27b2/A".to_owned(),
19408 kind: refs::SeedKind::Unresolved,
19409 sha: String::new(),
19410 branch: true,
19411 detail: "no branch or commit named magi/27b2/A".to_owned(),
19412 }];
19413
19414 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19415 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19416 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19417 let detail = empty_candidate_detail(&state, "main");
19418 assert!(detail.starts_with("empty candidate"), "{detail}");
19419 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19420 assert!(detail.contains("--start-from"), "{detail}");
19421 }
19422
19423 #[tokio::test]
19432 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19433 let tmp = tempfile::tempdir().expect("tempdir");
19434 let repo = tmp.path().join("repo");
19435 std::fs::create_dir_all(&repo).unwrap();
19436 init_repo(&repo);
19437
19438 let mut config = Config::default();
19439 config.merge.mode = MergeMode::Local;
19440
19441 let mut state = RunState::new(
19442 repo.clone(),
19443 "main".to_owned(),
19444 "deadbeef".to_owned(),
19445 "task".to_owned(),
19446 config,
19447 );
19448 state.candidates = vec![Candidate {
19449 index: 0,
19450 label: 'A',
19451 agent: "alpha".to_owned(),
19452 branch: "does-not-exist".to_owned(),
19453 worktree: repo.clone(),
19454 summary: String::new(),
19455 stat: String::new(),
19456 files: 0,
19457 commits: 0,
19458 empty: false,
19459 failed: None,
19460 verified_noop: None,
19461 duration_ms: 0,
19462 folded: false,
19463 }];
19464 state.tally = Some(Tally {
19465 first_choice: BTreeMap::from([('A', 1)]),
19466 borda: BTreeMap::new(),
19467 winner: 'A',
19468 rankings: 1,
19469 unanimous_initial: true,
19470 deliberated: false,
19471 changed_votes: 0,
19472 unanimous_final: true,
19473 tie_break: None,
19474 judges: 0,
19475 present: 0,
19476 quorum: 0,
19477 met_quorum: true,
19478 uncontested: Some("only candidate A produced a change".to_owned()),
19479 });
19480 state.reviews = vec![ReviewRound {
19481 round: 1,
19482 head: "deadbeef".to_owned(),
19483 verified_head: None,
19484 verified_at: None,
19485 reviews: Vec::new(),
19486 e2e: Vec::new(),
19487 fix: None,
19488 blocking: 0,
19489 answered: 0,
19490 expected: 0,
19491 clean: true,
19492 verify_retried: false,
19493 e2e_deferred: false,
19494 e2e_defer_reason: None,
19495 progressed: false,
19496 vote_split: false,
19497 reconsideration: Vec::new(),
19498 verdict: None,
19499 }];
19500 state.gate = vec![CommandOutcome {
19501 command: "test".to_owned(),
19502 code: Some(0),
19503 output_tail: String::new(),
19504 duration_ms: 0,
19505 resource_blocked: false,
19506 }];
19507 state.gate_ran = true;
19508 state.status = RunStatus::Ready;
19513 state.merge = Some(MergeOutcome {
19514 mode: MergeMode::Local,
19515 ok: false,
19516 detail: "already concluded".to_owned(),
19517 empty: false,
19518 });
19519
19520 let mut runner = Runner {
19521 state,
19522 roles: ResolvedRoles {
19523 implementers: Vec::new(),
19524 judges: Vec::new(),
19525 reviewers: Vec::new(),
19526 fixer: None,
19527 conductor: conductor(),
19528 implementer_roster: Vec::new(),
19529 judge_roster: Vec::new(),
19530 reviewer_roster: Vec::new(),
19531 },
19532 sem: Arc::new(Semaphore::new(1)),
19533 pause: Pause::new(),
19534 interrupt: Pause::new(),
19535 };
19536
19537 runner.merge().await.expect("merge");
19538
19539 assert_eq!(
19540 runner.state.status,
19541 RunStatus::Ready,
19542 "a concluded run's status must not change on reentry"
19543 );
19544 assert_eq!(
19545 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19546 Some("already concluded"),
19547 "merge must not run again once the node already recorded an outcome"
19548 );
19549 }
19550
19551 #[tokio::test]
19560 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19561 let tmp = tempfile::tempdir().expect("tempdir");
19562 let repo = tmp.path().join("repo");
19563 std::fs::create_dir_all(&repo).unwrap();
19564 init_repo(&repo);
19565
19566 let mut config = Config::default();
19567 config.merge.mode = MergeMode::Local;
19568
19569 let mut state = RunState::new(
19570 repo.clone(),
19571 "main".to_owned(),
19572 "deadbeef".to_owned(),
19573 "task".to_owned(),
19574 config,
19575 );
19576 state.candidates = vec![Candidate {
19577 index: 0,
19578 label: 'A',
19579 agent: "alpha".to_owned(),
19580 branch: "does-not-exist".to_owned(),
19581 worktree: repo.clone(),
19582 summary: String::new(),
19583 stat: String::new(),
19584 files: 0,
19585 commits: 0,
19586 empty: false,
19587 failed: None,
19588 verified_noop: None,
19589 duration_ms: 0,
19590 folded: false,
19591 }];
19592 state.tally = Some(Tally {
19593 first_choice: BTreeMap::from([('A', 1)]),
19594 borda: BTreeMap::new(),
19595 winner: 'A',
19596 rankings: 1,
19597 unanimous_initial: true,
19598 deliberated: false,
19599 changed_votes: 0,
19600 unanimous_final: true,
19601 tie_break: None,
19602 judges: 0,
19603 present: 0,
19604 quorum: 0,
19605 met_quorum: true,
19606 uncontested: Some("only candidate A produced a change".to_owned()),
19607 });
19608 state.reviews = vec![ReviewRound {
19609 round: 1,
19610 head: "deadbeef".to_owned(),
19611 verified_head: None,
19612 verified_at: None,
19613 reviews: Vec::new(),
19614 e2e: Vec::new(),
19615 fix: None,
19616 blocking: 0,
19617 answered: 0,
19618 expected: 0,
19619 clean: true,
19620 verify_retried: false,
19621 e2e_deferred: false,
19622 e2e_defer_reason: None,
19623 progressed: false,
19624 vote_split: false,
19625 reconsideration: Vec::new(),
19626 verdict: None,
19627 }];
19628 state.gate = Vec::new();
19630 state.gate_ran = false;
19631 state.status = RunStatus::Gating;
19632
19633 let mut runner = Runner {
19634 state,
19635 roles: ResolvedRoles {
19636 implementers: Vec::new(),
19637 judges: Vec::new(),
19638 reviewers: Vec::new(),
19639 fixer: None,
19640 conductor: conductor(),
19641 implementer_roster: Vec::new(),
19642 judge_roster: Vec::new(),
19643 reviewer_roster: Vec::new(),
19644 },
19645 sem: Arc::new(Semaphore::new(1)),
19646 pause: Pause::new(),
19647 interrupt: Pause::new(),
19648 };
19649
19650 runner.merge().await.expect("merge");
19651
19652 assert!(
19653 runner.state.merge.is_none(),
19654 "an empty gate must never be read as a passing one: {:?}",
19655 runner.state.merge
19656 );
19657 }
19658
19659 #[tokio::test]
19666 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19667 let tmp = tempfile::tempdir().expect("tempdir");
19668 let repo = tmp.path().join("repo");
19669 std::fs::create_dir_all(&repo).unwrap();
19670 init_repo(&repo);
19671
19672 let config = Config::default();
19674
19675 let mut state = RunState::new(
19676 repo.clone(),
19677 "main".to_owned(),
19678 "deadbeef".to_owned(),
19679 "task".to_owned(),
19680 config,
19681 );
19682 state.candidates = vec![Candidate {
19683 index: 0,
19684 label: 'A',
19685 agent: "alpha".to_owned(),
19686 branch: "does-not-exist".to_owned(),
19687 worktree: repo.clone(),
19688 summary: String::new(),
19689 stat: String::new(),
19690 files: 0,
19691 commits: 0,
19692 empty: false,
19693 failed: None,
19694 verified_noop: None,
19695 duration_ms: 0,
19696 folded: false,
19697 }];
19698 state.tally = Some(Tally {
19699 first_choice: BTreeMap::from([('A', 1)]),
19700 borda: BTreeMap::new(),
19701 winner: 'A',
19702 rankings: 1,
19703 unanimous_initial: true,
19704 deliberated: false,
19705 changed_votes: 0,
19706 unanimous_final: true,
19707 tie_break: None,
19708 judges: 0,
19709 present: 0,
19710 quorum: 0,
19711 met_quorum: true,
19712 uncontested: Some("only candidate A produced a change".to_owned()),
19713 });
19714 state.reviews = vec![ReviewRound {
19715 round: 1,
19716 head: "deadbeef".to_owned(),
19717 verified_head: None,
19718 verified_at: None,
19719 reviews: Vec::new(),
19720 e2e: Vec::new(),
19721 fix: None,
19722 blocking: 0,
19723 answered: 0,
19724 expected: 0,
19725 clean: true,
19726 verify_retried: false,
19727 e2e_deferred: false,
19728 e2e_defer_reason: None,
19729 progressed: false,
19730 vote_split: false,
19731 reconsideration: Vec::new(),
19732 verdict: None,
19733 }];
19734
19735 let mut runner = Runner {
19736 state,
19737 roles: ResolvedRoles {
19738 implementers: Vec::new(),
19739 judges: Vec::new(),
19740 reviewers: Vec::new(),
19741 fixer: None,
19742 conductor: conductor(),
19743 implementer_roster: Vec::new(),
19744 judge_roster: Vec::new(),
19745 reviewer_roster: Vec::new(),
19746 },
19747 sem: Arc::new(Semaphore::new(1)),
19748 pause: Pause::new(),
19749 interrupt: Pause::new(),
19750 };
19751
19752 runner.gate().await.expect("gate");
19753 assert!(
19754 runner.state.gate_ran,
19755 "zero configured commands is still a real attempt, not an unrun gate"
19756 );
19757 assert!(runner.state.gate.is_empty());
19758 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19759 assert_ne!(
19760 runner.state.status,
19761 RunStatus::Blocked,
19762 "a gate with nothing to check must not read as failed"
19763 );
19764
19765 runner.merge().await.expect("merge");
19766 assert_eq!(
19767 runner.state.status,
19768 RunStatus::Ready,
19769 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19770 );
19771 }
19772
19773 #[tokio::test]
19784 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19785 crate::run::pin_test_home();
19786 let home = crate::run::home();
19787
19788 let tmp = tempfile::tempdir().expect("tempdir");
19789 let repo = tmp.path().join("repo");
19790 std::fs::create_dir_all(&repo).unwrap();
19791 init_repo(&repo);
19792 let cache_dir = tmp.path().join("target");
19795
19796 let mut config = Config::default();
19797 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19798 config.graph.timeout_verify = Some(2);
19802 config.graph.timeout_resource_wait = 0;
19803
19804 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19805 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19806 .expect("no io error acquiring directly")
19807 {
19808 crate::cache::AcquireOutcome::Acquired(g) => g,
19809 crate::cache::AcquireOutcome::Busy(b) => {
19810 panic!("expected the direct acquire to win the lease first: {b:?}")
19811 }
19812 };
19813
19814 let mut state = RunState::new(
19815 repo.clone(),
19816 "main".to_owned(),
19817 "deadbeef".to_owned(),
19818 "task".to_owned(),
19819 config,
19820 );
19821 state.candidates = vec![Candidate {
19822 index: 0,
19823 label: 'A',
19824 agent: "alpha".to_owned(),
19825 branch: "does-not-exist".to_owned(),
19826 worktree: repo.clone(),
19827 summary: String::new(),
19828 stat: String::new(),
19829 files: 0,
19830 commits: 0,
19831 empty: false,
19832 failed: None,
19833 verified_noop: None,
19834 duration_ms: 0,
19835 folded: false,
19836 }];
19837 state.tally = Some(Tally {
19838 first_choice: BTreeMap::from([('A', 1)]),
19839 borda: BTreeMap::new(),
19840 winner: 'A',
19841 rankings: 1,
19842 unanimous_initial: true,
19843 deliberated: false,
19844 changed_votes: 0,
19845 unanimous_final: true,
19846 tie_break: None,
19847 judges: 0,
19848 present: 0,
19849 quorum: 0,
19850 met_quorum: true,
19851 uncontested: Some("only candidate A produced a change".to_owned()),
19852 });
19853 state.reviews = vec![ReviewRound {
19854 round: 1,
19855 head: "deadbeef".to_owned(),
19856 verified_head: None,
19857 verified_at: None,
19858 reviews: Vec::new(),
19859 e2e: Vec::new(),
19860 fix: None,
19861 blocking: 0,
19862 answered: 0,
19863 expected: 0,
19864 clean: true,
19865 verify_retried: false,
19866 e2e_deferred: false,
19867 e2e_defer_reason: None,
19868 progressed: false,
19869 vote_split: false,
19870 reconsideration: Vec::new(),
19871 verdict: None,
19872 }];
19873
19874 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19875 state.reviews[0].e2e = vec![CommandOutcome {
19876 command: state.config.verify.e2e[0].clone(),
19877 code: Some(0),
19878 output_tail: String::new(),
19879 duration_ms: 0,
19880 resource_blocked: false,
19881 }];
19882 let mut runner = Runner {
19883 state,
19884 roles: ResolvedRoles {
19885 implementers: Vec::new(),
19886 judges: Vec::new(),
19887 reviewers: Vec::new(),
19888 fixer: None,
19889 conductor: conductor(),
19890 implementer_roster: Vec::new(),
19891 judge_roster: Vec::new(),
19892 reviewer_roster: Vec::new(),
19893 },
19894 sem: Arc::new(Semaphore::new(1)),
19895 pause: Pause::new(),
19896 interrupt: Pause::new(),
19897 };
19898
19899 runner.gate().await.expect("gate");
19900 assert!(runner.state.resource_block.is_none());
19901 assert!(
19902 !runner
19903 .state
19904 .events
19905 .iter()
19906 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19907 "a gate with nothing to run must never request the held cache lease"
19908 );
19909 assert!(
19910 runner.state.gate_ran,
19911 "zero commands is still a real, immediate attempt"
19912 );
19913 assert!(runner.state.gate.is_empty());
19914 assert_ne!(
19915 runner.state.status,
19916 RunStatus::Blocked,
19917 "must not read as resource-blocked on a lease it never asked for"
19918 );
19919 }
19920
19921 #[cfg(unix)]
19924 #[tokio::test]
19925 async fn verification_is_standalone_inside_a_live_worker() {
19926 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19927 if std::env::var_os(CHILD).is_none() {
19928 let home = tempfile::tempdir().unwrap();
19929 let generation = crate::worker::pin_current(home.path()).unwrap();
19930 let output = std::process::Command::new(std::env::current_exe().unwrap())
19931 .args([
19932 "--exact",
19933 "graph::tests::verification_is_standalone_inside_a_live_worker",
19934 "--nocapture",
19935 ])
19936 .env(CHILD, "1")
19937 .env("MAGI_HOME", home.path())
19938 .env("MAGI_WORKER_HOME", home.path())
19939 .env("MAGI_WORKER_ID", "abcdef")
19940 .env("MAGI_WORKER_GENERATION", &generation.id)
19941 .env("MAGI_WORKER_EXE", &generation.executable)
19942 .env("MAGI_RUN", "verification-parent")
19943 .env("MAGI_NODE", "gate")
19944 .env("MAGI_SEAT", "worker-fixture")
19945 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19946 .env("MAGI_NO_AUTOUPDATE", "0")
19947 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19948 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19949 .env(
19950 "VERIFY_EXPECT_TOOLCHAIN",
19951 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19952 )
19953 .env(
19954 "VERIFY_EXPECT_CACHE",
19955 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19956 )
19957 .output()
19958 .unwrap();
19959 assert!(
19960 output.status.success()
19961 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19962 "{}\n{}",
19963 String::from_utf8_lossy(&output.stdout),
19964 String::from_utf8_lossy(&output.stderr)
19965 );
19966 return;
19967 }
19968 let home = crate::run::home();
19969 let directory = home.join("workers/jobs/abcdef");
19970 std::fs::create_dir_all(&directory).unwrap();
19971 let job = crate::worker::Job {
19972 id: "abcdef".into(),
19973 sequence: 1,
19974 kind: "task".into(),
19975 key: "task:verification".into(),
19976 payload: serde_json::json!({}),
19977 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19978 version: Some(env!("CARGO_PKG_VERSION").into()),
19979 status: crate::worker::Status::Running,
19980 result: None,
19981 pid: Some(std::process::id()),
19982 created_at: 0,
19983 finished_at: None,
19984 };
19985 std::fs::write(
19986 directory.join("job.json"),
19987 serde_json::to_vec(&job).unwrap(),
19988 )
19989 .unwrap();
19990 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19991 let assert_worker = || {
19992 assert_eq!(
19993 crate::worker::current_job(&home).unwrap().unwrap().id,
19994 job.id
19995 );
19996 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19997 assert_eq!(
19998 crate::worker::helper_executable(&home).unwrap(),
19999 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20000 );
20001 assert!(
20002 crate::updater::Checker::new(&crate::config::Update {
20003 mode: crate::config::UpdateMode::Notify,
20004 interval: None,
20005 auto_restart: false,
20006 })
20007 .is_none(),
20008 "the driver remains pinned, even while tests are standalone"
20009 );
20010 };
20011 assert_worker();
20012 let repo = home.join("repo");
20013 std::fs::create_dir_all(&repo).unwrap();
20014 init_repo(&repo);
20015 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20018 test -z "${MAGI_WORKER_GENERATION+x}" &&
20019 test -z "${MAGI_WORKER_HOME+x}" &&
20020 test -z "${MAGI_WORKER_EXE+x}" &&
20021 test "$MAGI_HOME/repo" -ef . &&
20022 test "$MAGI_RUN" = verification-parent &&
20023 test "$MAGI_NODE" = gate &&
20024 test "$MAGI_SEAT" = worker-fixture &&
20025 test -n "$MAGI_PROCESS_OWNER" &&
20026 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20027 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20028 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20029 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20030 assert!(
20032 !std::process::Command::new("sh")
20033 .args(["-c", probe])
20034 .current_dir(&repo)
20035 .status()
20036 .unwrap()
20037 .success()
20038 );
20039 let unrelated_home = tempfile::tempdir().unwrap();
20040 assert!(
20041 crate::worker::current_job(unrelated_home.path()).is_err(),
20042 "an inherited worker id cannot be resolved in a unit fixture's home"
20043 );
20044 let mut runner = runner_at(RunStatus::Gating);
20045 runner.state.repo = repo.clone();
20046 runner.state.config.disk.min_free_bytes = 0;
20047 runner.state.config.coordination = crate::config::test_coordination();
20048 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20049 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20050 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20051 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20052 let winner = Candidate {
20053 index: 0,
20054 label: 'A',
20055 agent: "fixture".into(),
20056 branch: "main".into(),
20057 worktree: repo.clone(),
20058 summary: String::new(),
20059 stat: String::new(),
20060 files: 0,
20061 commits: 0,
20062 empty: false,
20063 failed: None,
20064 verified_noop: None,
20065 duration_ms: 0,
20066 folded: false,
20067 };
20068 let (e2e, retried, pids) = run_e2e_with_retry(
20069 &mut runner.state,
20070 &["sh".into(), "-c".into()],
20071 &[format!("{probe} && printf initial")],
20072 &repo,
20073 Duration::from_secs(10),
20074 "initial",
20075 )
20076 .await;
20077 assert!(!retried && pids.is_empty());
20078 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20079 assert_eq!(e2e[0].output_tail, "initial");
20080 let gate = runner.run_gate(&winner).await.unwrap();
20081 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20082 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20085 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20086 passed: false,
20087 outcomes: Vec::new(),
20088 });
20089 runner.state.save().unwrap();
20090 let mut resumed = RunState::load(&runner.state.id).unwrap();
20091 let wait = crate::repo_lock::WaitPolicy::default();
20092 assert!(matches!(
20093 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20094 .await
20095 .unwrap(),
20096 GateFix::Retry
20097 ));
20098 let saved = RunState::load(&resumed.id).unwrap();
20099 let repair = saved.gate_repair_e2e.unwrap();
20100 assert!(repair.passed);
20101 assert_eq!(repair.outcomes.len(), 1);
20102 assert!(repair.outcomes[0].ok());
20103 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20104 .await
20105 .unwrap();
20106 assert_eq!(landing.len(), 1);
20107 assert!(landing[0].ok(), "{landing:?}");
20108 assert_eq!(
20109 std::fs::read_to_string(repo.join("phases")).unwrap(),
20110 "pre\ngate\ne2e\npre\ngate\n"
20111 );
20112 assert_worker();
20113 }
20114
20115 #[cfg(unix)]
20116 #[tokio::test]
20117 async fn verification_keeps_the_panic_and_the_full_output() {
20118 crate::run::pin_test_home();
20119 let tmp = tempfile::tempdir().unwrap();
20120 let mut config = Config::default();
20121 config.disk.min_free_bytes = 0;
20122 let mut state = RunState::new(
20123 tmp.path().to_owned(),
20124 "main".into(),
20125 "base".into(),
20126 "verification evidence".into(),
20127 config,
20128 );
20129 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20130 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20131 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20132 let mut names = Vec::new();
20133 for _ in 0..2 {
20134 let (outcomes, _) = run_commands(
20135 &mut state,
20136 "gate",
20137 "gate",
20138 0,
20139 &["sh".into(), "-c".into()],
20140 &[script.into()],
20141 tmp.path(),
20142 Duration::from_secs(60),
20143 )
20144 .await;
20145 let tail = &outcomes[0].output_tail;
20146 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20147 assert!(tail.contains("boom assertion"));
20148 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20149 let line = tail.lines().next().unwrap();
20150 names.push(line.to_owned());
20151 }
20152 assert_ne!(names[0], names[1], "the second run must not overwrite");
20153 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20154 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20155 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20156
20157 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20159 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20160 let (outcomes, _) = run_commands(
20161 &mut state,
20162 "gate",
20163 "gate",
20164 0,
20165 &["sh".into(), "-c".into()],
20166 &["echo failure-output; exit 7".into()],
20167 tmp.path(),
20168 Duration::from_secs(10),
20169 )
20170 .await;
20171 assert_eq!(outcomes[0].code, Some(7));
20172 assert!(state.events.iter().any(|e| {
20173 e.node == "gate" && e.message.contains("could not save the full gate output:")
20174 }));
20175 }
20176
20177 #[cfg(unix)]
20178 #[tokio::test]
20179 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20180 crate::run::pin_test_home();
20181 let tmp = tempfile::tempdir().unwrap();
20182 let mut config = Config::default();
20183 config.disk.min_free_bytes = 0;
20184 let mut state = RunState::new(
20185 tmp.path().to_owned(),
20186 "main".into(),
20187 "base".into(),
20188 "verification evidence".into(),
20189 config,
20190 );
20191 let (outcomes, pids) = run_commands(
20192 &mut state,
20193 "gate",
20194 "gate",
20195 1,
20196 &["sh".into(), "-c".into()],
20197 &[
20198 "printf before-timeout; exec sleep 30".into(),
20199 "kill -TERM $$".into(),
20200 "exit 7".into(),
20201 "printf success".into(),
20202 ],
20203 tmp.path(),
20204 Duration::from_secs(1),
20205 )
20206 .await;
20207 wait_for_timed_out_children_to_die(&pids).await;
20208 assert_eq!(
20209 pids.len(),
20210 1,
20211 "unleased timeout retains its death-check PID"
20212 );
20213 assert_eq!(outcomes.len(), 4);
20214 assert_eq!(outcomes[0].code, None);
20215 assert!(outcomes[0].output_tail.contains("before-timeout"));
20216 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20217 assert_eq!(outcomes[1].code, None);
20218 assert!(outcomes[1].output_tail.contains("command terminated:"));
20219 assert!(!outcomes[1].output_tail.contains("timed out"));
20220 assert_eq!(outcomes[2].code, Some(7));
20221 assert!(!outcomes[2].output_tail.contains("timed out"));
20222 assert_eq!(outcomes[3].code, Some(0));
20223 assert_eq!(outcomes[3].output_tail, "success");
20224 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20225 }
20226
20227 #[tokio::test]
20237 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20238 crate::run::pin_test_home();
20239
20240 let tmp = tempfile::tempdir().expect("tempdir");
20241 let repo = tmp.path().join("repo");
20242 std::fs::create_dir_all(&repo).unwrap();
20243 init_repo(&repo);
20244
20245 let mut config = Config::default();
20246 config.disk.min_free_bytes = 0;
20247 config.verify.gate = vec![
20248 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20249 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20250 .to_owned(),
20251 ];
20252
20253 let mut state = RunState::new(
20254 repo.clone(),
20255 "main".to_owned(),
20256 "deadbeef".to_owned(),
20257 "task".to_owned(),
20258 config,
20259 );
20260 let run_id = state.id.clone();
20261 state.candidates = vec![Candidate {
20262 index: 0,
20263 label: 'A',
20264 agent: "alpha".to_owned(),
20265 branch: "does-not-exist".to_owned(),
20266 worktree: repo.clone(),
20267 summary: String::new(),
20268 stat: String::new(),
20269 files: 0,
20270 commits: 0,
20271 empty: false,
20272 failed: None,
20273 verified_noop: None,
20274 duration_ms: 0,
20275 folded: false,
20276 }];
20277 state.tally = Some(Tally {
20278 first_choice: BTreeMap::from([('A', 1)]),
20279 borda: BTreeMap::new(),
20280 winner: 'A',
20281 rankings: 1,
20282 unanimous_initial: true,
20283 deliberated: false,
20284 changed_votes: 0,
20285 unanimous_final: true,
20286 tie_break: None,
20287 judges: 0,
20288 present: 0,
20289 quorum: 0,
20290 met_quorum: true,
20291 uncontested: Some("only candidate A produced a change".to_owned()),
20292 });
20293 state.reviews = vec![ReviewRound {
20294 round: 1,
20295 head: "deadbeef".to_owned(),
20296 verified_head: None,
20297 verified_at: None,
20298 reviews: Vec::new(),
20299 e2e: Vec::new(),
20300 fix: None,
20301 blocking: 0,
20302 answered: 0,
20303 expected: 0,
20304 clean: true,
20305 verify_retried: false,
20306 e2e_deferred: false,
20307 e2e_defer_reason: None,
20308 progressed: false,
20309 vote_split: false,
20310 reconsideration: Vec::new(),
20311 verdict: None,
20312 }];
20313
20314 let mut runner = Runner {
20315 state,
20316 roles: ResolvedRoles {
20317 implementers: Vec::new(),
20318 judges: Vec::new(),
20319 reviewers: Vec::new(),
20320 fixer: None,
20321 conductor: conductor(),
20322 implementer_roster: Vec::new(),
20323 judge_roster: Vec::new(),
20324 reviewer_roster: Vec::new(),
20325 },
20326 sem: Arc::new(Semaphore::new(1)),
20327 pause: Pause::new(),
20328 interrupt: Pause::new(),
20329 };
20330
20331 let started_marker = repo.join("started.marker");
20332 let release_marker = repo.join("release.marker");
20333 let poller = tokio::spawn(async move {
20334 for _ in 0..100 {
20339 if started_marker.exists()
20340 && let Ok(s) = crate::run::RunState::load(&run_id)
20341 && let Some(a) = s.active.get("gate")
20342 {
20343 std::fs::write(&release_marker, b"go").expect("release marker");
20344 return Some(a.clone());
20345 }
20346 tokio::time::sleep(Duration::from_millis(50)).await;
20347 }
20348 None
20349 });
20350
20351 runner.gate().await.expect("gate");
20352 let captured = poller.await.expect("poller task");
20353 let captured = captured.expect(
20354 "the poller never saw a `gate` task entry in run.json while the command was \
20355 still blocked on its own release marker",
20356 );
20357
20358 assert_eq!(captured.task.as_deref(), Some("gate"));
20359 assert_eq!(captured.node, "gate");
20360 assert_eq!(captured.index, Some(1));
20361 assert_eq!(captured.total, Some(1));
20362 assert!(
20363 captured
20364 .command
20365 .as_deref()
20366 .is_some_and(|c| c.contains("started.marker")),
20367 "{captured:?}"
20368 );
20369
20370 assert!(
20371 runner.state.active.is_empty(),
20372 "the entry must be cleared once the command actually finished: {:?}",
20373 runner.state.active
20374 );
20375 assert!(runner.state.gate_ran);
20376 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20377 }
20378
20379 #[tokio::test]
20382 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20383 use crate::verdict::{Finding, ReviewVote, Severity};
20384 crate::run::pin_test_home();
20385 let tmp = tempfile::tempdir().expect("tempdir");
20386 let repo = tmp.path().join("repo");
20387 std::fs::create_dir_all(&repo).unwrap();
20388 init_repo(&repo);
20389
20390 for (severity, vote, expect) in [
20391 (Severity::Major, ReviewVote::Reject, true),
20392 (Severity::Minor, ReviewVote::Reject, false),
20393 (Severity::Major, ReviewVote::Approve, false),
20394 ] {
20395 let mut round = review_round(false, 1, 1, 1, false, true);
20396 round.reviews = vec![ReviewRecord {
20397 reviewer: 1,
20398 agent: "alpha".to_owned(),
20399 summary: String::new(),
20400 findings: vec![Finding {
20401 id: "R1-1-1".to_owned(),
20402 severity,
20403 file: None,
20404 line: None,
20405 title: "t".to_owned(),
20406 detail: String::new(),
20407 }],
20408 vote: Some(vote),
20409 failed: None,
20410 duration_ms: 0,
20411 attempts: 0,
20412 }];
20413 let mut state = RunState::new(
20414 repo.clone(),
20415 "main".to_owned(),
20416 "deadbeef".to_owned(),
20417 "task".to_owned(),
20418 Config::default(),
20419 );
20420 state.reviews = vec![round];
20421 let mut runner = Runner {
20422 state,
20423 roles: ResolvedRoles {
20424 implementers: Vec::new(),
20425 judges: Vec::new(),
20426 reviewers: Vec::new(),
20427 fixer: None,
20428 conductor: conductor(),
20429 implementer_roster: Vec::new(),
20430 judge_roster: Vec::new(),
20431 reviewer_roster: Vec::new(),
20432 },
20433 sem: Arc::new(Semaphore::new(1)),
20434 pause: Pause::new(),
20435 interrupt: Pause::new(),
20436 };
20437 let shell = runner.state.config.shell();
20438 runner
20439 .stop_reviewing("round budget spent", &shell, &repo)
20440 .await
20441 .expect("stop_reviewing");
20442 assert_eq!(runner.state.status, RunStatus::Gating);
20443 assert_eq!(
20444 runner.state.contested_handoff.is_some(),
20445 expect,
20446 "{severity:?} + {vote:?}"
20447 );
20448 }
20449 }
20450
20451 #[tokio::test]
20464 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20465 crate::run::pin_test_home();
20466 let home = crate::run::home();
20467
20468 let tmp = tempfile::tempdir().expect("tempdir");
20469 let repo = tmp.path().join("repo");
20470 std::fs::create_dir_all(&repo).unwrap();
20471 init_repo(&repo);
20472 let head = crate::git::rev_parse(&repo, "HEAD")
20473 .await
20474 .expect("rev-parse");
20475 let cache_dir = tmp.path().join("target");
20478
20479 let mut config = Config::default();
20480 config.verify.e2e = vec![format!(
20481 "CARGO_TARGET_DIR='{}' test -f README.md",
20482 cache_dir.display()
20483 )];
20484 config.graph.review_rounds = 1;
20485 config.graph.timeout_verify = Some(2);
20488 config.graph.timeout_resource_wait = 2;
20489
20490 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20491 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20492 .expect("no io error acquiring directly")
20493 {
20494 crate::cache::AcquireOutcome::Acquired(g) => g,
20495 crate::cache::AcquireOutcome::Busy(b) => {
20496 panic!("expected the direct acquire to win the lease first: {b:?}")
20497 }
20498 };
20499
20500 let mut state = RunState::new(
20501 repo.clone(),
20502 "main".to_owned(),
20503 head.clone(),
20504 "task".to_owned(),
20505 config,
20506 );
20507 state.candidates = vec![Candidate {
20508 index: 0,
20509 label: 'A',
20510 agent: "alpha".to_owned(),
20511 branch: "does-not-exist".to_owned(),
20512 worktree: repo.clone(),
20513 summary: String::new(),
20514 stat: String::new(),
20515 files: 0,
20516 commits: 0,
20517 empty: false,
20518 failed: None,
20519 verified_noop: None,
20520 duration_ms: 0,
20521 folded: false,
20522 }];
20523 state.tally = Some(Tally {
20524 first_choice: BTreeMap::from([('A', 1)]),
20525 borda: BTreeMap::new(),
20526 winner: 'A',
20527 rankings: 1,
20528 unanimous_initial: true,
20529 deliberated: false,
20530 changed_votes: 0,
20531 unanimous_final: true,
20532 tie_break: None,
20533 judges: 0,
20534 present: 0,
20535 quorum: 0,
20536 met_quorum: true,
20537 uncontested: Some("only candidate A produced a change".to_owned()),
20538 });
20539 state.reviews = vec![ReviewRound {
20543 round: 1,
20544 head: head.clone(),
20545 verified_head: None,
20546 verified_at: None,
20547 reviews: Vec::new(),
20548 e2e: Vec::new(),
20549 fix: None,
20550 blocking: 1,
20551 answered: 1,
20552 expected: 1,
20553 clean: false,
20554 verify_retried: false,
20555 e2e_deferred: true,
20556 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20557 progressed: false,
20558 vote_split: false,
20559 reconsideration: Vec::new(),
20560 verdict: None,
20561 }];
20562
20563 let mut runner = Runner {
20564 state,
20565 roles: ResolvedRoles {
20566 implementers: Vec::new(),
20567 judges: Vec::new(),
20568 reviewers: Vec::new(),
20569 fixer: None,
20570 conductor: conductor(),
20571 implementer_roster: Vec::new(),
20572 judge_roster: Vec::new(),
20573 reviewer_roster: Vec::new(),
20574 },
20575 sem: Arc::new(Semaphore::new(1)),
20576 pause: Pause::new(),
20577 interrupt: Pause::new(),
20578 };
20579
20580 let shell = runner.state.config.shell();
20581 runner
20582 .stop_reviewing("round budget spent", &shell, &repo)
20583 .await
20584 .expect("stop_reviewing");
20585
20586 let last = runner.state.reviews.last().expect("round record");
20587 assert_eq!(
20588 last.e2e_status(),
20589 E2eStatus::ResourceBlocked,
20590 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20591 failed: {last:?}"
20592 );
20593 assert_eq!(
20594 last.verified_head.as_deref(),
20595 Some(head.as_str()),
20596 "which commit this attempt targeted is known even though nothing finished checking \
20597 it"
20598 );
20599 let first_attempt_at = last
20600 .verified_at
20601 .expect("when this attempt ran is known too");
20602 assert_ne!(
20603 runner.state.status,
20604 RunStatus::Blocked,
20605 "contention is evidence about the machine, not the patch — it must not settle the \
20606 run as blocked: {:?}",
20607 runner.state.status
20608 );
20609 assert!(
20610 !runner
20611 .state
20612 .events
20613 .iter()
20614 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20615 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20616 runner.state.events
20617 );
20618
20619 runner
20624 .stop_reviewing("round budget spent", &shell, &repo)
20625 .await
20626 .expect("stop_reviewing retry");
20627 assert_eq!(
20628 runner.state.reviews.len(),
20629 1,
20630 "no new round was started: {:?}",
20631 runner.state.reviews
20632 );
20633 let last = runner.state.reviews.last().expect("round record");
20634 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20635 assert!(
20636 last.verified_at.expect("still known") > first_attempt_at,
20637 "a second reentry must be a fresh attempt, not a stale copy of the first"
20638 );
20639 assert_ne!(runner.state.status, RunStatus::Blocked);
20640
20641 held.release();
20642 }
20643
20644 #[cfg(unix)]
20645 #[tokio::test]
20646 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20647 ask_test_home();
20648 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20649 let tmp = tempfile::tempdir().unwrap();
20650 let repo = tmp.path().join("repo");
20651 std::fs::create_dir(&repo).unwrap();
20652 init_repo(&repo);
20653 let run_git = |args: &[&str]| {
20654 let out = std::process::Command::new("git")
20655 .args(args)
20656 .current_dir(&repo)
20657 .output()
20658 .unwrap();
20659 assert!(
20660 out.status.success(),
20661 "{args:?}: {}",
20662 String::from_utf8_lossy(&out.stderr)
20663 );
20664 };
20665 let origin = tmp.path().join("origin.git");
20666 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20667 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20668 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20669 run_git(&["add", "spec.rs"]);
20670 run_git(&["commit", "-m", "initial spec"]);
20671 run_git(&["push", "origin", "main"]);
20672 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20673 run_git(&["checkout", "-b", "fix/review"]);
20674 std::fs::write(
20675 repo.join("check.rs"),
20676 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20677 )
20678 .unwrap();
20679 run_git(&["add", "check.rs"]);
20680 run_git(&["commit", "-m", "use spec"]);
20681 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20682 run_git(&["checkout", "main"]);
20683 std::fs::write(
20684 repo.join("spec.rs"),
20685 "struct AgentSpec { max_turn_secs: u64 }\n",
20686 )
20687 .unwrap();
20688 run_git(&["add", "spec.rs"]);
20689 run_git(&["commit", "-m", "extend spec"]);
20690 run_git(&["push", "origin", "main"]);
20691 run_git(&["checkout", "fix/review"]);
20692
20693 let mut runner = runner_at(RunStatus::Reviewing);
20694 runner.state.repo = repo.clone();
20695 runner.state.base_commit = base;
20696 runner.state.pr_head = Some("fix/review".into());
20697 runner.state.config.graph.review_rounds = max_rounds;
20698 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20699 runner.state.config.graph.sessions = false;
20700 runner.state.config.disk.min_free_bytes = 0;
20701 runner.state.config.verify.e2e =
20702 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20703 runner.state.candidates = vec![Candidate {
20704 index: 0,
20705 label: 'A',
20706 agent: "fixture".into(),
20707 branch: "fix/review".into(),
20708 worktree: repo.clone(),
20709 summary: String::new(),
20710 stat: String::new(),
20711 files: 1,
20712 commits: 1,
20713 empty: false,
20714 failed: None,
20715 verified_noop: None,
20716 duration_ms: 0,
20717 folded: false,
20718 }];
20719 runner.state.tally = Some(Tally {
20720 first_choice: BTreeMap::new(),
20721 borda: BTreeMap::new(),
20722 winner: 'A',
20723 rankings: 0,
20724 unanimous_initial: true,
20725 deliberated: false,
20726 changed_votes: 0,
20727 unanimous_final: true,
20728 tie_break: None,
20729 judges: 0,
20730 present: 0,
20731 quorum: 0,
20732 met_quorum: true,
20733 uncontested: Some("review-only".into()),
20734 });
20735 let mut last = review_round(false, 0, 1, 1, false, false);
20736 last.head = reviewed.clone();
20737 last.e2e_deferred = true;
20738 last.e2e[0].resource_blocked = true;
20739 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20740 if max_rounds == 1 {
20741 last.e2e.clear();
20742 }
20743 runner.state.reviews = vec![last];
20744 let calls = tmp.path().join("fix-calls");
20745 let mut fixer = spec("repair");
20746 fixer
20747 .env
20748 .insert("FIX_CALLS".into(), calls.display().to_string());
20749 fixer.command = vec!["sh".into(), "-c".into(), r#"
20750 echo fix >> "$FIX_CALLS"
20751 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20752 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20753 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20754 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20755 "#.into()];
20756 let mut reviewer = spec("reviewer");
20757 reviewer.command = vec![
20758 "sh".into(),
20759 "-c".into(),
20760 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20761 .into(),
20762 ];
20763 runner.roles.fixer = Some(vec![fixer]);
20764 runner.roles.reviewers = vec![reviewer];
20765
20766 runner.sync_to_base().await.unwrap();
20767 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20768 assert_ne!(rebased, reviewed);
20769 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20770 if reentry {
20771 let shell = runner.state.config.shell();
20774 runner
20775 .stop_unresolved_review("deferred verification", &shell, &repo)
20776 .await
20777 .unwrap();
20778 assert!(runner.catchup_repairable());
20779 assert!(!calls.exists());
20780 runner.state = RunState::load(&runner.state.id).unwrap();
20781 assert!(runner.catchup_repairable());
20782 }
20783 runner.review_loop().await.unwrap();
20784 let first = &runner.state.reviews[0];
20785 assert_eq!(first.head, reviewed);
20786 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20787 assert!(
20788 first.verify_retried,
20789 "a reproduced build failure gets exactly one retry: {:?}",
20790 first.e2e
20791 );
20792 assert!(first.e2e[0].output_tail.contains("E0063"));
20793 assert!(!runner.catchup_repairable());
20794 if max_rounds == 2 {
20795 assert_eq!(
20796 runner.state.status,
20797 RunStatus::Gating,
20798 "{}",
20799 crate::report::run(&runner.state)
20800 );
20801 assert_eq!(runner.state.reviews.len(), 2);
20802 assert!(first.fix.as_ref().unwrap().committed);
20803 assert!(runner.state.reviews[1].clean);
20804 assert_eq!(
20805 runner
20806 .state
20807 .reviews
20808 .iter()
20809 .filter(|r| r.fix.is_some())
20810 .count(),
20811 1
20812 );
20813 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20814 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20815 } else {
20816 assert_eq!(runner.state.status, RunStatus::Blocked);
20817 assert!(first.fix.is_none());
20818 assert!(!calls.exists());
20819 let recovery = crate::recovery::assess(
20820 "prior",
20821 Some(&runner.state),
20822 &runner.landing_base(),
20823 crate::recovery::Relation::Diverged,
20824 );
20825 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20826 let reasons = recovery.reasons.join("\n");
20827 assert!(
20828 reasons.contains(&reviewed) && reasons.contains(&rebased),
20829 "{reasons}"
20830 );
20831 assert!(
20832 reasons.contains("base sync to")
20833 && reasons.contains("additional review is required"),
20834 "{reasons}"
20835 );
20836 }
20837 }
20838 }
20839
20840 #[tokio::test]
20852 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20853 crate::run::pin_test_home();
20854 let home = crate::run::home();
20855
20856 let tmp = tempfile::tempdir().expect("tempdir");
20857 let repo = tmp.path().join("repo");
20858 std::fs::create_dir_all(&repo).unwrap();
20859 init_repo(&repo);
20860 let head = crate::git::rev_parse(&repo, "HEAD")
20861 .await
20862 .expect("rev-parse");
20863 let cache_dir = tmp.path().join("target");
20864
20865 let mut config = Config::default();
20866 config.verify.e2e = vec![format!(
20867 "CARGO_TARGET_DIR='{}' test -f README.md",
20868 cache_dir.display()
20869 )];
20870 config.graph.review_rounds = 1;
20871 config.graph.timeout_verify = Some(2);
20872 config.graph.timeout_resource_wait = 2;
20873
20874 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20875 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20876 .expect("no io error acquiring directly")
20877 {
20878 crate::cache::AcquireOutcome::Acquired(g) => g,
20879 crate::cache::AcquireOutcome::Busy(b) => {
20880 panic!("expected the direct acquire to win the lease first: {b:?}")
20881 }
20882 };
20883
20884 let mut state = RunState::new(
20885 repo.clone(),
20886 "main".to_owned(),
20887 head.clone(),
20888 "task".to_owned(),
20889 config,
20890 );
20891 state.candidates = vec![Candidate {
20892 index: 0,
20893 label: 'A',
20894 agent: "alpha".to_owned(),
20895 branch: "does-not-exist".to_owned(),
20896 worktree: repo.clone(),
20897 summary: String::new(),
20898 stat: String::new(),
20899 files: 0,
20900 commits: 0,
20901 empty: false,
20902 failed: None,
20903 verified_noop: None,
20904 duration_ms: 0,
20905 folded: false,
20906 }];
20907 state.tally = Some(Tally {
20908 first_choice: BTreeMap::from([('A', 1)]),
20909 borda: BTreeMap::new(),
20910 winner: 'A',
20911 rankings: 1,
20912 unanimous_initial: true,
20913 deliberated: false,
20914 changed_votes: 0,
20915 unanimous_final: true,
20916 tie_break: None,
20917 judges: 0,
20918 present: 0,
20919 quorum: 0,
20920 met_quorum: true,
20921 uncontested: Some("only candidate A produced a change".to_owned()),
20922 });
20923 state.reviews = vec![ReviewRound {
20927 round: 1,
20928 head: head.clone(),
20929 verified_head: Some(head.clone()),
20930 verified_at: Some(jiff::Timestamp::now()),
20931 reviews: Vec::new(),
20932 e2e: vec![CommandOutcome {
20933 command: format!(
20934 "CARGO_TARGET_DIR='{}' test -f README.md",
20935 cache_dir.display()
20936 ),
20937 code: None,
20938 output_tail: "waiting for the shared build cache".to_owned(),
20939 duration_ms: 0,
20940 resource_blocked: true,
20941 }],
20942 fix: None,
20943 blocking: 1,
20944 answered: 1,
20945 expected: 1,
20946 clean: false,
20947 verify_retried: false,
20948 e2e_deferred: false,
20949 e2e_defer_reason: None,
20950 progressed: false,
20951 vote_split: false,
20952 reconsideration: Vec::new(),
20953 verdict: None,
20954 }];
20955
20956 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20957 let mut runner = Runner {
20958 state,
20959 roles: ResolvedRoles {
20960 implementers: Vec::new(),
20961 judges: Vec::new(),
20962 reviewers: Vec::new(),
20963 fixer: None,
20964 conductor: conductor(),
20965 implementer_roster: Vec::new(),
20966 judge_roster: Vec::new(),
20967 reviewer_roster: Vec::new(),
20968 },
20969 sem: Arc::new(Semaphore::new(1)),
20970 pause: Pause::new(),
20971 interrupt: Pause::new(),
20972 };
20973
20974 runner.review_loop().await.expect("review_loop");
20979
20980 assert_eq!(
20981 runner.state.reviews.len(),
20982 1,
20983 "no new round was started on top of the unresolved one: {:?}",
20984 runner.state.reviews
20985 );
20986 let last = &runner.state.reviews[0];
20987 assert_eq!(
20988 last.e2e_status(),
20989 E2eStatus::ResourceBlocked,
20990 "still contended: {last:?}"
20991 );
20992 assert!(
20993 last.verified_at.expect("still known") > first_attempt_at,
20994 "review_loop must have actually retried the check, not left it exactly as found"
20995 );
20996 assert_ne!(
20997 runner.state.status,
20998 RunStatus::Blocked,
20999 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21000 runner.state.status
21001 );
21002
21003 held.release();
21004 }
21005
21006 #[tokio::test]
21007 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21008 crate::run::pin_test_home();
21009 for status in [RunStatus::Blocked, RunStatus::Gating] {
21010 let mut runner = runner_at(status);
21011 runner.state.config.graph.land = true;
21012 runner.state.gate_ran = true;
21013 runner.state.merge = Some(MergeOutcome {
21014 mode: MergeMode::Pr,
21015 ok: false,
21016 detail: "the fixer produced no commit".into(),
21017 empty: false,
21018 });
21019 let url = "https://example.invalid/o/r/pull/1";
21020 runner.state.pr = Some(crate::run::PrRecord {
21021 url: url.into(),
21022 number: 1,
21023 state: "open".into(),
21024 checks: "red".into(),
21025 round: 1,
21026 rounds: 1,
21027 red_at_merge: Vec::new(),
21028 });
21029 runner.state.save().unwrap();
21030 runner.execute_graph().await.unwrap();
21034 assert_eq!(runner.state.status, RunStatus::Blocked);
21035 assert!(runner.state.land_sync_pending);
21036 assert!(runner.state.events.iter().any(|event|
21037 event.node == "land" && event.message == format!("watching {url}")));
21038 assert_eq!(
21039 runner.state.merge.as_ref().unwrap().detail,
21040 "the fixer produced no commit"
21041 );
21042 }
21043 }
21044
21045 #[tokio::test]
21046 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21047 crate::run::pin_test_home();
21048 let tmp = tempfile::tempdir().expect("tempdir");
21049 let repo = tmp.path().join("repo");
21050 std::fs::create_dir_all(&repo).unwrap();
21051 init_repo(&repo);
21052
21053 let mut config = Config::default();
21054 config.merge.mode = MergeMode::Pr;
21055 config.graph.land = true;
21056 config.graph.land_approval = false;
21057
21058 let mut state = RunState::new(
21059 repo.clone(),
21060 "main".to_owned(),
21061 "deadbeef".to_owned(),
21062 "task".to_owned(),
21063 config,
21064 );
21065 state.candidates = vec![Candidate {
21066 index: 0,
21067 label: 'A',
21068 agent: "alpha".to_owned(),
21069 branch: "does-not-exist".to_owned(),
21070 worktree: repo.clone(),
21071 summary: String::new(),
21072 stat: String::new(),
21073 files: 0,
21074 commits: 0,
21075 empty: false,
21076 failed: None,
21077 verified_noop: None,
21078 duration_ms: 0,
21079 folded: false,
21080 }];
21081 state.tally = Some(Tally {
21082 first_choice: BTreeMap::from([('A', 1)]),
21083 borda: BTreeMap::new(),
21084 winner: 'A',
21085 rankings: 1,
21086 unanimous_initial: true,
21087 deliberated: false,
21088 changed_votes: 0,
21089 unanimous_final: true,
21090 tie_break: None,
21091 judges: 0,
21092 present: 0,
21093 quorum: 0,
21094 met_quorum: true,
21095 uncontested: Some("only candidate A produced a change".to_owned()),
21096 });
21097 state.reviews = vec![ReviewRound {
21098 round: 1,
21099 head: "deadbeef".to_owned(),
21100 verified_head: None,
21101 verified_at: None,
21102 reviews: Vec::new(),
21103 e2e: Vec::new(),
21104 fix: None,
21105 blocking: 0,
21106 answered: 0,
21107 expected: 0,
21108 clean: true,
21109 verify_retried: false,
21110 e2e_deferred: false,
21111 e2e_defer_reason: None,
21112 progressed: false,
21113 vote_split: false,
21114 reconsideration: Vec::new(),
21115 verdict: None,
21116 }];
21117 state.gate = vec![CommandOutcome {
21118 command: "test".to_owned(),
21119 code: Some(0),
21120 output_tail: String::new(),
21121 duration_ms: 0,
21122 resource_blocked: false,
21123 }];
21124 state.gate_ran = true;
21125 state.status = RunStatus::Landing;
21129 state.merge = Some(MergeOutcome {
21130 mode: MergeMode::Pr,
21131 ok: true,
21132 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21133 empty: false,
21134 });
21135
21136 ask_test_home();
21140 state.save().expect("save landing run");
21143 let store = ask::Questions::open();
21144 let q = ask_open_question(&store, &state.id);
21145
21146 let mut runner = Runner {
21147 state,
21148 roles: ResolvedRoles {
21149 implementers: Vec::new(),
21150 judges: Vec::new(),
21151 reviewers: Vec::new(),
21152 fixer: None,
21153 conductor: conductor(),
21154 implementer_roster: Vec::new(),
21155 judge_roster: Vec::new(),
21156 reviewer_roster: Vec::new(),
21157 },
21158 sem: Arc::new(Semaphore::new(1)),
21159 pause: Pause::new(),
21160 interrupt: Pause::new(),
21161 };
21162
21163 runner.execute().await.expect("execute");
21168
21169 assert_eq!(
21170 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21171 Some("https://example.invalid/x/y/pull/1"),
21172 "reentry must not push again or open a second pull request over the \
21173 one `land` is already watching"
21174 );
21175 assert_ne!(
21176 runner.state.status,
21177 RunStatus::Landing,
21178 "land could not actually reach the fake pull request, so it must \
21179 have given up rather than left the run silently parked forever"
21180 );
21181 assert_eq!(runner.state.status, RunStatus::Blocked);
21185 assert!(
21186 store.get(&q.id).unwrap().status.open(),
21187 "Blocked is still alive; settle_questions must have been a no-op here"
21188 );
21189 }
21190
21191 fn state_with_round(round: ReviewRound) -> RunState {
21192 let mut s = RunState::new(
21193 PathBuf::from("/repo"),
21194 "main".to_owned(),
21195 "abc1234".to_owned(),
21196 "add retries".to_owned(),
21197 Config::default(),
21198 );
21199 s.reviews = vec![round];
21200 s
21201 }
21202
21203 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21204 crate::verdict::Finding {
21205 id: id.to_owned(),
21206 severity,
21207 file: None,
21208 line: None,
21209 title: title.to_owned(),
21210 detail: String::new(),
21211 }
21212 }
21213
21214 #[test]
21215 fn pr_body_names_open_findings_and_declined_ones() {
21216 let round = ReviewRound {
21217 round: 2,
21218 head: "deadbee".to_owned(),
21219 verified_head: None,
21220 verified_at: None,
21221 reviews: vec![ReviewRecord {
21222 attempts: 0,
21223 reviewer: 1,
21224 agent: "alpha".to_owned(),
21225 summary: String::new(),
21226 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21227 vote: None,
21228 failed: None,
21229 duration_ms: 0,
21230 }],
21231 e2e: vec![CommandOutcome {
21232 command: "cargo test".to_owned(),
21233 code: Some(0),
21234 output_tail: String::new(),
21235 duration_ms: 0,
21236 resource_blocked: false,
21237 }],
21238 verify_retried: false,
21239 e2e_deferred: false,
21240 e2e_defer_reason: None,
21241 fix: Some(FixRecord {
21242 agent: "alpha".to_owned(),
21243 addressed: Vec::new(),
21244 rejected: vec![crate::verdict::Rejection {
21245 id: "R1-1-1".to_owned(),
21246 why: "not reachable from any caller".to_owned(),
21247 }],
21248 notes: String::new(),
21249 committed: true,
21250 failed: None,
21251 duration_ms: 0,
21252 continuation: None,
21253 }),
21254 blocking: 0,
21255 answered: 1,
21256 expected: 1,
21257 clean: false,
21258 progressed: true,
21259 vote_split: false,
21260 reconsideration: Vec::new(),
21261 verdict: None,
21262 };
21263 let state = state_with_round(round);
21264 let body = pr_message(&state, 'A').body;
21265
21266 assert!(body.contains("add retries"), "the task must still be there");
21267 assert!(body.contains("R2-1-1"), "{body}");
21268 assert!(!body.contains("unused import"), "{body}");
21269 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21270 assert!(
21271 !body.contains("not reachable from any caller"),
21272 "the reason it was declined: {body}"
21273 );
21274 }
21275
21276 #[test]
21277 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21278 let round = ReviewRound {
21279 round: 1,
21280 head: "deadbee".to_owned(),
21281 verified_head: None,
21282 verified_at: None,
21283 reviews: vec![ReviewRecord {
21284 attempts: 0,
21285 reviewer: 1,
21286 agent: "alpha".to_owned(),
21287 summary: String::new(),
21288 findings: Vec::new(),
21289 vote: None,
21290 failed: None,
21291 duration_ms: 0,
21292 }],
21293 e2e: Vec::new(),
21294 verify_retried: false,
21295 e2e_deferred: false,
21296 e2e_defer_reason: None,
21297 fix: None,
21298 blocking: 0,
21299 answered: 1,
21300 expected: 1,
21301 clean: true,
21302 progressed: false,
21303 vote_split: false,
21304 reconsideration: Vec::new(),
21305 verdict: None,
21306 };
21307 let state = state_with_round(round);
21308 let body = pr_message(&state, 'A').body;
21309 assert!(!body.contains("Open review findings"), "{body}");
21310 assert!(!body.contains("Declined"), "{body}");
21311 }
21312
21313 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21314 let mut state = RunState::new(
21315 PathBuf::from("/repo"),
21316 "main".to_owned(),
21317 "abc1234".to_owned(),
21318 instruction.to_owned(),
21319 Config::default(),
21320 );
21321 state.config.disk.min_free_bytes = 0;
21323 state.candidates.push(Candidate {
21324 index: 0,
21325 label: 'A',
21326 agent: "alpha".to_owned(),
21327 branch: "magi/x/A".to_owned(),
21328 worktree: PathBuf::from("/wt"),
21329 summary: summary.to_owned(),
21330 stat: String::new(),
21331 files: 1,
21332 commits: 1,
21333 empty: false,
21334 failed: None,
21335 verified_noop: None,
21336 folded: false,
21337 duration_ms: 0,
21338 });
21339 state
21340 }
21341
21342 fn review_state(subjects: &[&str]) -> RunState {
21343 let mut state = state_with_summary(
21344 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21345 "",
21346 );
21347 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21348 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21349 state
21350 }
21351
21352 #[tokio::test]
21355 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21356 ask_test_home();
21357 let tmp = tempfile::tempdir().unwrap();
21358 let repo = tmp.path().join("repo");
21359 std::fs::create_dir_all(&repo).unwrap();
21360 init_repo(&repo);
21361 let origin = tmp.path().join("origin.git");
21362 let g = |dir: &Path, args: &[&str]| {
21363 let out = std::process::Command::new("git")
21364 .args(args)
21365 .current_dir(dir)
21366 .quiet()
21367 .output()
21368 .expect("spawn git");
21369 assert!(
21370 out.status.success(),
21371 "git {args:?}: {}",
21372 String::from_utf8_lossy(&out.stderr)
21373 );
21374 };
21375 g(
21376 tmp.path(),
21377 &[
21378 "clone",
21379 "--bare",
21380 "-q",
21381 repo.to_str().unwrap(),
21382 origin.to_str().unwrap(),
21383 ],
21384 );
21385 g(
21386 &repo,
21387 &["remote", "add", "origin", origin.to_str().unwrap()],
21388 );
21389 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21390
21391 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21393 g(&repo, &["add", "-A"]);
21394 g(
21395 &repo,
21396 &["commit", "-q", "-m", "chore(deps): update a crate"],
21397 );
21398 g(&repo, &["push", "-q", "origin", "main"]);
21399 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21400 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21401 g(&repo, &["add", "-A"]);
21402 g(
21403 &repo,
21404 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21405 );
21406 g(&repo, &["checkout", "-q", "main"]);
21407
21408 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21409 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21410 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21412 assert_eq!(fallback, c1);
21413
21414 let mut state = review_state(&[
21415 "chore(deps): update a crate",
21416 "fix(daemon): apply a chosen action",
21417 ]);
21418 state.repo = repo.clone();
21419 state.base_branch = "main".to_owned();
21420 state.base_commit = c1;
21421 refresh_reviewed_commits(&mut state, "feat/own").await;
21422 assert_eq!(
21423 state.reviewed_commits,
21424 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21425 );
21426 assert_eq!(
21427 review_title(&state).as_deref(),
21428 Some("fix(daemon): apply a chosen action")
21429 );
21430 assert_eq!(
21431 leaked_subjects(&state, "feat/own").await,
21432 Some(vec!["chore(deps): update a crate".to_owned()])
21433 );
21434 state.config.merge.remote = "nowhere".to_owned();
21437 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21438 }
21439
21440 struct SummaryFixture {
21444 _tmp: tempfile::TempDir,
21445 repo: PathBuf,
21446 reply: PathBuf,
21447 runner: Runner,
21448 }
21449
21450 fn summary_fixture() -> SummaryFixture {
21451 ask_test_home();
21452 let tmp = tempfile::tempdir().unwrap();
21453 let repo = tmp.path().join("repo");
21454 std::fs::create_dir_all(&repo).unwrap();
21455 init_repo(&repo);
21456 let origin = tmp.path().join("origin.git");
21457 let g = |args: &[&str]| {
21458 let out = std::process::Command::new("git")
21459 .args(args)
21460 .current_dir(&repo)
21461 .quiet()
21462 .output()
21463 .expect("spawn git");
21464 assert!(
21465 out.status.success(),
21466 "git {args:?}: {}",
21467 String::from_utf8_lossy(&out.stderr)
21468 );
21469 };
21470 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21471 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21472 g(&["push", "-q", "origin", "main"]);
21473 g(&["checkout", "-q", "-b", "feat/own"]);
21474 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21475 g(&["add", "-A"]);
21476 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21477 let reply = tmp.path().join("reply.txt");
21478 std::fs::write(
21479 &reply,
21480 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21481 )
21482 .unwrap();
21483 let mut runner = runner_at(RunStatus::Gating);
21484 let mut state = review_state(&["feat(own): add the own file"]);
21485 state.repo = repo.clone();
21486 state.base_branch = "main".to_owned();
21487 state.candidates[0].branch = "feat/own".to_owned();
21488 state.candidates[0].worktree = repo.clone();
21489 let mut author = spec("alpha");
21490 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21491 author
21492 .env
21493 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21494 state.config.agents = vec![author];
21495 runner.state = state;
21496 SummaryFixture {
21497 _tmp: tmp,
21498 repo,
21499 reply,
21500 runner,
21501 }
21502 }
21503
21504 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21505 let winner = f.runner.state.candidates[0].clone();
21506 f.runner
21507 .guarded_pr_message(&winner, None, true)
21508 .await
21509 .unwrap()
21510 .expect("an English description never parks")
21511 }
21512
21513 fn clean_round_with_minor_findings() -> ReviewRound {
21514 let mut round = passing_round(2, None);
21515 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21516 round.reviews = vec![ReviewRecord {
21517 attempts: 0,
21518 reviewer: 1,
21519 agent: "alpha".into(),
21520 summary: "内部レビュー".into(),
21521 findings: vec![
21522 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21523 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21524 ],
21525 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21526 failed: None,
21527 duration_ms: 0,
21528 }];
21529 round
21530 }
21531
21532 #[tokio::test]
21533 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21534 for review_only in [false, true] {
21535 let mut f = summary_fixture();
21536 if !review_only {
21537 f.runner.state.candidates[0].agent = "alpha".into();
21538 }
21539 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21540 assert!(f.runner.state.reviews[0].clean);
21541 assert!(f.runner.state.reviews[0].fix.is_none());
21542 assert!(f.runner.state.open_findings().is_empty());
21543 assert_eq!(
21544 crate::pr_outcome::required_finding_ids(&f.runner.state),
21545 ["R2-1-1", "R2-1-2"]
21546 );
21547 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21548 let evidence = crate::pr_outcome::summary_evidence(
21549 &f.runner.state,
21550 crate::pr_outcome::FinalHead::At(&head),
21551 );
21552 assert!(
21553 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21554 );
21555 let omitted = posted(&mut f).await;
21557 assert!(omitted.body.contains("No description"));
21558 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21559 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21560 assert!(f.runner.state.change_summary.is_none());
21561 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21562 let described = posted(&mut f).await;
21563 assert!(described.body.contains("improve documentation for callers"));
21564 assert!(described.body.contains("remaining edge case"));
21565 assert!(described.body.contains("## Open review findings"));
21566 assert!(!described.body.contains("未修正"));
21567 }
21568 }
21569
21570 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21571 let mut f = summary_fixture();
21572 if !review_only {
21573 f.runner.state.candidates[0].agent = "alpha".into();
21574 }
21575 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21576 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21577 fixed
21578 .fix
21579 .as_mut()
21580 .unwrap()
21581 .rejected
21582 .push(crate::verdict::Rejection {
21583 id: "R1-1-1".into(),
21584 why: "呼び出されないコードのため辞退".into(),
21585 });
21586 let mut clean = clean_round_with_minor_findings();
21587 clean.verified_head = Some(head.clone());
21588 f.runner.state.reviews = vec![fixed, clean];
21589 ok_gate(&mut f.runner.state, Some(&head));
21590 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21591 f.runner.state.config.agents.push(scripted("beta", reply));
21592 f.runner.state.config.roles.language_judge =
21593 Some(crate::config::AgentChoice::One("beta".into()));
21594 f
21595 }
21596
21597 #[tokio::test]
21598 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21599 for review_only in [false, true] {
21600 let mut f = rewrite_fixture_with_findings(review_only,
21601 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21602 let m = posted(&mut f).await;
21603 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21604 let expected =
21605 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21606 assert!(m.body.contains(&expected));
21607 assert!(m.body.contains("## Review rounds"));
21608 assert!(
21609 m.body
21610 .contains("Gate (`verify.gate`): passed on the final commit")
21611 );
21612 assert!(
21613 m.body
21614 .contains("Tests (`verify.e2e`): passed on the final commit")
21615 );
21616 let c = f.runner.state.change_summary.as_ref().unwrap();
21617 assert!(
21618 m.body
21619 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21620 );
21621 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21622 assert!(crate::pr_outcome::covers_required_findings(
21623 &f.runner.state,
21624 &m.body
21625 ));
21626 assert!(m.body.contains("documentation remains incomplete"));
21627 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21628 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21629 assert!(f.runner.state.github_text.is_none());
21630 }
21631 }
21632
21633 #[tokio::test]
21634 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21635 for review_only in [false, true] {
21636 for reply in [
21637 "TITLE: fix: public description\nA short English description.",
21638 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21639 ] {
21640 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21641 let winner = f.runner.state.candidates[0].clone();
21642 let posted = f
21643 .runner
21644 .guarded_pr_message(&winner, None, true)
21645 .await
21646 .unwrap();
21647 assert!(posted.is_none());
21648 assert!(f.runner.state.parked);
21649 let withheld = f.runner.state.github_text.as_ref().unwrap();
21650 assert!(
21651 withheld.rewrite.is_none(),
21652 "an incomplete rewrite must not be selectable"
21653 );
21654 let q = ask::Questions::open()
21655 .get(withheld.question.as_ref().unwrap())
21656 .unwrap();
21657 assert!(
21658 !q.choices
21659 .contains(&crate::github_text::USE_REWRITE.to_owned())
21660 );
21661 assert!(
21662 q.detail.contains("R1-1-1")
21663 && q.detail.contains("R2-1-1")
21664 && q.detail.contains("R2-1-2")
21665 );
21666 }
21667 }
21668 }
21669
21670 #[tokio::test]
21671 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21672 for review_only in [false, true] {
21673 let mut f = summary_fixture();
21674 if !review_only {
21675 f.runner.state.candidates[0].agent = "alpha".into();
21676 }
21677 f.runner.state.instruction = "内部タスクの説明".into();
21678 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21679 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21680 f.runner.state.reviews = vec![
21681 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21682 round_with_notes(2, Some("二回目の内部修正報告です。")),
21683 passing_round(3, Some(&head)),
21684 ];
21685 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21686 ok_gate(&mut f.runner.state, Some(&head));
21687 let captured = f.repo.join("public-prompt.txt");
21688 let author = &mut f.runner.state.config.agents[0];
21689 author.command = vec![
21690 "sh".into(),
21691 "-c".into(),
21692 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21693 ];
21694 author.env.insert(
21695 "PROMPT_FILE".into(),
21696 captured.to_string_lossy().into_owned(),
21697 );
21698 let m = posted(&mut f).await;
21699 assert!(m.body.contains("Adds `own.txt`"));
21700 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21701 assert!(m.body.contains("R1-1-1"));
21702 assert!(
21703 m.body
21704 .contains("Gate (`verify.gate`): passed on the final commit")
21705 );
21706 assert!(
21707 m.body
21708 .contains("Tests (`verify.e2e`): passed on the final commit")
21709 );
21710 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21711 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21712 let prompt = std::fs::read_to_string(captured).unwrap();
21713 assert!(prompt.contains(&format!(
21714 "git diff {} {head}",
21715 f.runner.state.change_summary.as_ref().unwrap().base
21716 )));
21717 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21718 assert_eq!(
21719 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21720 "二回目の内部修正報告です。"
21721 );
21722 }
21723 }
21724
21725 #[tokio::test]
21726 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21727 for review_only in [false, true] {
21728 let mut f = summary_fixture();
21729 if !review_only {
21730 f.runner.state.candidates[0].agent = "alpha".into();
21731 }
21732 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21733 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21734 std::fs::write(&f.reply, "no usable description").unwrap();
21735 let failed = posted(&mut f).await;
21736 assert!(
21737 failed
21738 .body
21739 .contains("No description of what this change does could be generated")
21740 );
21741 assert!(failed.body.is_ascii());
21742 assert!(f.runner.state.github_text.is_none());
21743 assert!(f.runner.state.change_summary.is_none());
21744 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21745 let evidence = crate::pr_outcome::summary_evidence(
21746 &f.runner.state,
21747 crate::pr_outcome::FinalHead::At(&head),
21748 );
21749 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21750 assert!(
21751 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21752 .exists()
21753 );
21754 f.runner.state =
21755 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21756 std::fs::write(
21757 &f.reply,
21758 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21759 )
21760 .unwrap();
21761 let retried = posted(&mut f).await;
21762 assert!(retried.body.contains("Adds the committed own file."));
21763 assert_eq!(
21764 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21765 "日本語の修正報告"
21766 );
21767 }
21768 }
21769
21770 #[tokio::test]
21771 async fn same_head_verification_and_findings_invalidate_public_prose() {
21772 let mut f = summary_fixture();
21773 posted(&mut f).await;
21774 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21775 let original = f
21776 .runner
21777 .state
21778 .change_summary
21779 .as_ref()
21780 .unwrap()
21781 .evidence
21782 .clone();
21783 ok_gate(&mut f.runner.state, Some(&head));
21784 std::fs::write(
21785 &f.reply,
21786 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21787 )
21788 .unwrap();
21789 let verified = posted(&mut f).await;
21790 assert!(verified.body.contains("now verified"));
21791 assert_ne!(
21792 f.runner.state.change_summary.as_ref().unwrap().evidence,
21793 original
21794 );
21795 let mut round = round_with_notes(1, Some("内部報告"));
21796 round.reviews.push(ReviewRecord {
21797 attempts: 0,
21798 reviewer: 1,
21799 agent: "alpha".into(),
21800 summary: "内部レビュー".into(),
21801 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21802 vote: None,
21803 failed: None,
21804 duration_ms: 0,
21805 });
21806 round
21807 .fix
21808 .as_mut()
21809 .unwrap()
21810 .rejected
21811 .push(crate::verdict::Rejection {
21812 id: "R1-1-1".into(),
21813 why: "呼び出し元から到達できない".into(),
21814 });
21815 f.runner.state.reviews = vec![round];
21816 let omitted = posted(&mut f).await;
21818 assert!(omitted.body.contains("No description"));
21819 assert!(omitted.body.contains("R1-1-1"));
21820 assert!(!omitted.body.contains("到達"));
21821 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21822 let described = posted(&mut f).await;
21823 assert!(described.body.contains("unreachable from callers"));
21824 assert!(described.body.contains("R1-1-1"));
21825 assert!(!described.body.contains("到達"));
21826 let evidence = f
21827 .runner
21828 .state
21829 .change_summary
21830 .as_ref()
21831 .unwrap()
21832 .evidence
21833 .clone();
21834 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21835 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21836 assert!(posted(&mut f).await.body.contains("rationale updated"));
21837 assert_ne!(
21838 f.runner.state.change_summary.as_ref().unwrap().evidence,
21839 evidence
21840 );
21841 f.runner
21843 .state
21844 .change_summary
21845 .as_mut()
21846 .unwrap()
21847 .evidence
21848 .clear();
21849 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21850 assert!(posted(&mut f).await.body.contains("current specification"));
21851 }
21852
21853 #[tokio::test]
21854 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21855 let mut f = summary_fixture();
21856 let m = posted(&mut f).await;
21857 assert_eq!(m.title, "feat(own): add the own file");
21858 assert!(
21859 m.body
21860 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21861 "{}",
21862 m.body
21863 );
21864 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21865 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21866 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21867 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21868 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21869 let kept = f.runner.state.change_summary.as_ref().unwrap();
21870 assert_eq!(
21871 (kept.head.as_str(), kept.base.as_str()),
21872 (head.as_str(), base.as_str())
21873 );
21874 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21875 }
21876
21877 #[tokio::test]
21878 async fn review_only_summary_receives_the_configured_github_language() {
21879 let mut f = summary_fixture();
21880 f.runner.state.config.graph.github_language = "ja".into();
21881 let captured = f.repo.join("summary-prompt.txt");
21882 let author = &mut f.runner.state.config.agents[0];
21883 author.command = vec![
21884 "sh".into(),
21885 "-c".into(),
21886 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21887 ];
21888 author.env.insert(
21889 "PROMPT_FILE".into(),
21890 captured.to_string_lossy().into_owned(),
21891 );
21892 std::fs::write(
21893 &f.reply,
21894 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21895 )
21896 .unwrap();
21897 let m = posted(&mut f).await;
21898 let prompt = std::fs::read_to_string(captured).unwrap();
21899 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21900 assert!(!prompt.contains("in English"));
21901 assert_eq!(m.title, "feat(own): ファイルを追加");
21902 assert!(m.body.contains("own.txt を追加します。"));
21903 }
21904
21905 #[tokio::test]
21906 async fn review_fixes_come_after_the_whole_change_summary() {
21907 let mut f = summary_fixture();
21908 f.runner.state.reviews = vec![
21909 round_with_notes(1, Some("handled the PATH case")),
21910 round_with_notes(2, None),
21911 ];
21912 let m = posted(&mut f).await;
21913 let summary = m.body.find("## Summary").expect("summary");
21914 let fixes = m.body.find("## Review rounds").expect("review rounds");
21915 assert!(summary < fixes, "{}", m.body);
21916 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21917 assert!(m.body.contains("## Final verification"), "{}", m.body);
21918 }
21919
21920 #[tokio::test]
21921 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21922 let mut f = summary_fixture();
21923 let first = posted(&mut f).await;
21924 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21925 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21927 let again = posted(&mut f).await;
21928 assert_eq!((again.title, again.body), (first.title, first.body));
21929
21930 let g = |args: &[&str]| {
21931 let out = std::process::Command::new("git")
21932 .args(args)
21933 .current_dir(&f.repo)
21934 .quiet()
21935 .output()
21936 .expect("spawn git");
21937 assert!(out.status.success(), "git {args:?}");
21938 };
21939 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21940 g(&["add", "-A"]);
21941 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21942 let second = posted(&mut f).await;
21943 let kept = f.runner.state.change_summary.clone().unwrap();
21944 assert_ne!(kept.head, first_head);
21945 assert!(second.body.contains("Not read yet."), "{}", second.body);
21946 assert!(!second.body.contains("so the first summary is read"));
21947 assert_eq!(second.title, "feat(own): other");
21948 }
21949
21950 #[tokio::test]
21951 async fn an_unavailable_summary_is_stated_and_never_invented() {
21952 let mut f = summary_fixture();
21953 std::fs::write(&f.reply, "no title line here").unwrap();
21954 let m = posted(&mut f).await;
21955 assert!(
21956 m.body
21957 .contains("No description of what this change does could be generated"),
21958 "{}",
21959 m.body
21960 );
21961 assert!(m.body.contains("internal reports remain in local records"));
21962 assert!(f.runner.state.change_summary.is_none());
21963 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21964 let winner = f.runner.state.candidates[0].clone();
21966 let preview = f
21967 .runner
21968 .guarded_pr_message(&winner, None, false)
21969 .await
21970 .unwrap()
21971 .unwrap();
21972 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21973 }
21974
21975 #[test]
21976 fn pr_message_review_single_commit_uses_its_subject() {
21977 let state = review_state(&["feat(nats): per-role user"]);
21978 let m = pr_message(&state, 'A');
21979 assert_eq!(m.title, "feat(nats): per-role user");
21980 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21981 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21982 }
21983
21984 #[test]
21985 fn pr_message_review_multi_commit_takes_the_oldest() {
21986 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21987 let m = pr_message(&state, 'A');
21988 assert_eq!(m.title, "feat: the change");
21989 for s in ["feat: the change", "fix: typo", "fix: again"] {
21990 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21991 }
21992 }
21993
21994 #[test]
21995 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21996 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21997 let mut state = review_state(&[first, "fix: later fixup"]);
21998 state.candidates[0].branch = "機能/ログイン".to_owned();
21999 let m = pr_message(&state, 'A');
22000 assert!(
22001 m.title.starts_with("chore: land candidate A of run"),
22002 "{}",
22003 m.title
22004 );
22005 }
22006 }
22007
22008 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22009 BranchFacts {
22010 commits: commits
22011 .iter()
22012 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22013 .collect(),
22014 stat: stat.to_owned(),
22015 }
22016 }
22017
22018 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22019 let mut r = review_round(true, 0, 1, 1, true, true);
22020 r.round = round;
22021 r.fix = notes.map(|n| FixRecord {
22022 agent: "fixer".to_owned(),
22023 addressed: Vec::new(),
22024 rejected: Vec::new(),
22025 notes: n.to_owned(),
22026 committed: true,
22027 failed: None,
22028 duration_ms: 0,
22029 continuation: None,
22030 });
22031 r
22032 }
22033
22034 #[test]
22035 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22036 let state = review_state(&["ignored"]);
22037 let f = facts(
22038 &[
22039 (
22040 "fix(login): resolve PATH on macOS",
22041 "Login shells skip rc files.",
22042 ),
22043 ("fix: address review", ""),
22044 ],
22045 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22046 );
22047 let m = pr_message_with(&state, 'A', Some(&f));
22048 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22049 assert!(
22050 m.body
22051 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22052 "{}",
22053 m.body
22054 );
22055 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22056 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22057 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22058 for banned in [
22059 "Review the work already",
22060 "no task statement",
22061 "Original task",
22062 ] {
22063 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22064 }
22065 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22066 }
22067
22068 #[test]
22069 fn pr_message_review_truncates_a_huge_first_commit_body() {
22070 let state = review_state(&["ignored"]);
22071 let f = facts(
22072 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22073 "s",
22074 );
22075 let m = pr_message_with(&state, 'A', Some(&f));
22076 assert!(m.body.len() < 30_000, "{}", m.body.len());
22077 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22078 assert!(
22079 m.body.contains("- ... 1 more commit(s)"),
22080 "{}",
22081 m.body.len()
22082 );
22083 assert!(m.body.ends_with("magi:candidate-a\n"));
22084 }
22085
22086 #[test]
22087 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22088 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22089 assert_eq!(m.title, "feat: x");
22090 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22091 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22092 assert!(!m.body.contains("no task statement"), "{}", m.body);
22093 }
22094
22095 #[test]
22096 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22097 let mut state = review_state(&["日本語の件名"]);
22098 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22099 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22100 state.candidates[0].branch = "機能/ログイン".to_owned();
22101 assert!(
22102 pr_message(&state, 'A')
22103 .title
22104 .starts_with("chore: land candidate A")
22105 );
22106 }
22107
22108 #[test]
22109 fn pr_message_review_fixes_survive_a_clean_final_round() {
22110 let mut state = review_state(&["feat: x"]);
22111 state.reviews = vec![
22112 round_with_notes(1, Some("handled the PATH case")),
22113 round_with_notes(2, None),
22114 ];
22115 let body = pr_message(&state, 'A').body;
22116 assert!(
22117 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22118 "{body}"
22119 );
22120 assert!(body.contains("### Round 1"), "{body}");
22121
22122 state.reviews = vec![
22123 round_with_notes(1, Some("first fix")),
22124 round_with_notes(2, Some("")),
22125 round_with_notes(3, Some("second fix")),
22126 round_with_notes(4, None),
22127 ];
22128 let body = pr_message(&state, 'A').body;
22129 assert!(
22130 body.contains("### Round 1") && !body.contains("first fix"),
22131 "{body}"
22132 );
22133 assert!(
22134 body.contains("### Round 3") && !body.contains("second fix"),
22135 "{body}"
22136 );
22137 assert!(body.contains("### Round 2"), "{body}");
22138 }
22139
22140 const FINAL: &str = "abcdef1234567890";
22141
22142 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22143 let mut r = round_with_notes(round, None);
22144 r.verified_head = verified.map(str::to_owned);
22145 r
22146 }
22147
22148 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22149 state.gate = vec![CommandOutcome {
22150 command: "gate".to_owned(),
22151 code: Some(0),
22152 output_tail: String::new(),
22153 duration_ms: 0,
22154 resource_blocked: false,
22155 }];
22156 state.gate_ran = true;
22157 state.gate_head = head.map(str::to_owned);
22158 }
22159
22160 fn body_at(state: &RunState) -> String {
22161 pr_message_raw(
22162 state,
22163 'A',
22164 None,
22165 crate::pr_outcome::FinalHead::At(FINAL),
22166 ChangeNote::NotAsked,
22167 )
22168 .body
22169 }
22170
22171 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22172
22173 #[test]
22174 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22175 let mut state = review_state(&["feat: x"]);
22176 state.reviews = vec![
22177 round_with_notes(1, Some(SANDBOXED)),
22178 passing_round(2, Some(FINAL)),
22179 ];
22180 ok_gate(&mut state, Some(FINAL));
22181 let body = body_at(&state);
22182 assert!(!body.contains("Handled the PATH case."), "{body}");
22183 assert!(!body.contains("could not commit"), "{body}");
22184 assert!(!body.contains("could not run"), "{body}");
22185 assert!(
22186 body.contains("The fix is committed on the branch"),
22187 "{body}"
22188 );
22189 assert!(!body.contains("sandbox"), "{body}");
22190 assert!(
22191 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22192 "{body}"
22193 );
22194 assert!(
22195 body.contains("Gate (`verify.gate`): passed on the final commit"),
22196 "{body}"
22197 );
22198 assert!(
22200 state.reviews[0]
22201 .fix
22202 .as_ref()
22203 .unwrap()
22204 .notes
22205 .contains("could not commit")
22206 );
22207 }
22208
22209 #[test]
22210 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22211 let mut state = review_state(&["feat: x"]);
22212 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22213 r1.fix.as_mut().unwrap().committed = false;
22214 let mut r2 = passing_round(2, Some(FINAL));
22215 r2.e2e[0].code = Some(1);
22216 state.reviews = vec![r1, r2];
22217 let body = body_at(&state);
22218 assert!(!body.contains("could not commit"), "{body}");
22219 assert!(!body.contains("could not run"), "{body}");
22220 assert!(body.contains("This round produced no commit"), "{body}");
22221 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22222 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22223 assert!(!body.contains("passed"), "{body}");
22224 }
22225
22226 #[test]
22227 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22228 let mut state = review_state(&["feat: x"]);
22229 state.reviews = vec![
22230 round_with_notes(1, Some(SANDBOXED)),
22231 passing_round(2, Some("0123456789abcdef")),
22232 ];
22233 ok_gate(&mut state, Some("0123456789abcdef"));
22234 let body = body_at(&state);
22235 assert!(!body.contains("HTTP tests could not run"), "{body}");
22236 assert!(!body.contains("could not commit"), "{body}");
22238 assert!(
22239 body.contains("passed on an earlier commit (`0123456`)"),
22240 "{body}"
22241 );
22242 assert!(!body.contains("passed on the final commit"), "{body}");
22243
22244 ok_gate(&mut state, None);
22246 let body = body_at(&state);
22247 assert!(
22248 body.contains("passed on a commit that was not recorded"),
22249 "{body}"
22250 );
22251 }
22252
22253 #[test]
22254 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22255 let state = state_with_summary(
22256 "add retries to the client",
22257 "TITLE: feat: retries\n\nDid it.",
22258 );
22259 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22260 assert_eq!(m.title, "feat: retries");
22261 assert!(
22262 m.body.contains("<summary>Original task</summary>"),
22263 "{}",
22264 m.body
22265 );
22266 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22267 assert!(
22268 m.body
22269 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22270 "{}",
22271 m.body
22272 );
22273 }
22274
22275 #[test]
22276 fn pr_message_review_bounds_a_long_english_subject() {
22277 let long = format!("feat: {}", "word ".repeat(100));
22278 let m = pr_message(&review_state(&[&long]), 'A');
22279 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22280 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22281 }
22282
22283 #[test]
22284 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22285 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22286 let m = pr_message(&state, 'A');
22287 assert_eq!(m.title, "add retries");
22288 assert!(m.body.contains("<summary>Original task</summary>"));
22289 assert!(!m.body.contains("Commits under review"));
22290 assert_eq!(landing_subject_source(&state), state.instruction);
22291 }
22292
22293 #[test]
22294 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22295 let state = review_state(&["feat: the change", "fix: typo"]);
22296 let source = landing_subject_source(&state);
22297 assert_eq!(land::merge_subject("", &source), "feat: the change");
22298 assert_eq!(
22299 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22300 "feat: the change"
22301 );
22302 assert_eq!(
22304 land::merge_subject("feat: renamed by hand", &source),
22305 "feat: renamed by hand"
22306 );
22307 let blank = review_state(&["日本語"]);
22308 assert!(
22309 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22310 );
22311 }
22312
22313 #[test]
22314 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22315 let state = review_state(&["feat: the change"]);
22316 let source = landing_subject_source(&state);
22317 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22318 assert_eq!(
22319 land::merge_subject(landing_title(&state, old), &source),
22320 "feat: the change"
22321 );
22322 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22323 let task = state_with_summary("add retries", "");
22324 assert_eq!(landing_title(&task, old), old);
22325 }
22326
22327 #[test]
22328 fn pr_message_describes_the_change_not_the_task() {
22329 let state = state_with_summary(
22330 "今回やってほしいこと: results projector を直す",
22331 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22332 );
22333 let m = pr_message(&state, 'A');
22334 assert_eq!(m.title, "fix(web): batch the runs list reads");
22335 assert!(
22336 m.body.starts_with("## Summary\n\n- reads run.json once"),
22337 "{}",
22338 m.body
22339 );
22340 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22341 let task_at = m.body.find("今回やってほしいこと").unwrap();
22342 let details_at = m.body.find("<details>").unwrap();
22343 assert!(
22344 details_at < task_at,
22345 "the task lives inside <details>: {}",
22346 m.body
22347 );
22348 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22349 assert!(m.body.contains("magi:candidate-a"));
22350 }
22351
22352 #[test]
22353 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22354 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22355 let m = pr_message(&state, 'A');
22356 assert_eq!(m.title, "add retries");
22357 assert!(
22358 m.body.contains("## Summary\n\n- did some things"),
22359 "{}",
22360 m.body
22361 );
22362
22363 let none = RunState::new(
22364 PathBuf::from("/repo"),
22365 "main".to_owned(),
22366 "abc1234".to_owned(),
22367 "add retries".to_owned(),
22368 Config::default(),
22369 );
22370 let m = pr_message(&none, 'A');
22371 assert_eq!(m.title, "add retries");
22372 assert!(!m.body.contains("## Summary"), "{}", m.body);
22373 }
22374
22375 #[test]
22376 fn pr_message_refuses_the_candidate_commit_subject() {
22377 for bad in [
22378 "TITLE: magi: candidate A (uncommitted work)",
22379 "TITLE: chore: stuff (uncommitted work)",
22380 "TITLE: ",
22381 ] {
22382 let state = state_with_summary("add retries", bad);
22383 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22384 }
22385 }
22386
22387 #[test]
22388 fn pr_message_bounds_a_very_long_task_and_title() {
22389 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22390 let state = state_with_summary(&long, "- nothing");
22391 let m = pr_message(&state, 'A');
22392 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22393 assert!(!m.title.contains('\n'));
22394
22395 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22396 let m = pr_message(&state, 'A');
22397 assert!(m.title.starts_with("feat: "));
22398 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22399 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22400 }
22401
22402 fn long_title_of(instruction: &str) -> String {
22403 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22404 }
22405
22406 #[test]
22407 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22408 let first = "Make the landing path keep a readable title for long tasks";
22409 let line = format!(
22410 "{first}. {}",
22411 "And then keep going with more words ".repeat(20)
22412 );
22413 let t = long_title_of(&line);
22414 assert_eq!(t, first);
22415 assert!(!t.starts_with("chore: land"));
22416 }
22417
22418 #[test]
22419 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22420 let line = "word ".repeat(200);
22421 let t = long_title_of(&line);
22422 assert!(t.ends_with("word..."), "{t}");
22423 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22424 }
22425
22426 #[test]
22427 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22428 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22429 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22430 }
22431 }
22432
22433 #[test]
22434 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22435 let mut state = state_with_summary("再試行を追加する", "- nothing");
22436 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22437 state.config.graph.github_language = "ja".into();
22438 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22439 }
22440
22441 #[test]
22442 fn pr_message_title_limit_is_exact() {
22443 let at = "a".repeat(PR_TITLE_MAX);
22444 assert_eq!(long_title_of(&at), at);
22445 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22446 assert!(over.ends_with("..."), "{over}");
22447 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22448 }
22449
22450 #[test]
22451 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22452 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22453 let t = long_title_of(&line);
22454 assert!(t.ends_with("..."), "{t}");
22455 assert!(t.is_ascii(), "{t}");
22456 }
22457
22458 #[test]
22459 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22460 let line = format!(
22461 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22462 "plus more filler words ".repeat(20)
22463 );
22464 let t = long_title_of(&line);
22465 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22466 }
22467
22468 #[test]
22469 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22470 let line = format!("\n\n# {}", "title words ".repeat(40));
22471 let state = state_with_summary(&line, "- nothing");
22472 let m = pr_message(&state, 'A');
22473 assert!(m.title.starts_with("title words"), "{}", m.title);
22474 assert_eq!(
22475 land::merge_subject(&m.title, &landing_subject_source(&state)),
22476 m.title
22477 );
22478 assert_eq!(
22480 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22481 "feat: renamed by hand"
22482 );
22483 }
22484
22485 #[tokio::test]
22486 async fn github_text_rewrite_is_bounded_and_falls_back() {
22487 crate::run::pin_test_home();
22488 for (reply, accepted) in [
22489 (
22490 "TITLE: fix: retries\nAdd retries for failed requests.",
22491 true,
22492 ),
22493 (
22494 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22495 false,
22496 ),
22497 (
22498 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22499 true,
22500 ),
22501 (
22502 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22503 true,
22504 ),
22505 (
22506 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22507 true,
22508 ),
22509 (
22510 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22511 false,
22512 ),
22513 ("TITLE: fix: retries\n", false),
22514 ] {
22515 let dir = tempfile::tempdir().unwrap();
22516 let mut runner = runner_at(RunStatus::Gating);
22517 runner.state = state_with_summary(
22518 "add retries",
22519 "TITLE: fix: retries\n日本語の説明を書きます。",
22520 );
22521 runner.state.repo = dir.path().to_owned();
22522 runner.state.candidates[0].worktree = dir.path().to_owned();
22523 let mut author = spec("alpha");
22524 author.command = vec![
22525 "sh".into(),
22526 "-c".into(),
22527 "printf '%s' \"$REWRITE_REPLY\"".into(),
22528 ];
22529 author.env.insert("REWRITE_REPLY".into(), reply.into());
22530 runner.state.config.agents = vec![author];
22531 seed_public_summary(&mut runner);
22532 let winner = runner.state.candidates[0].clone();
22533 let first = runner
22534 .guarded_pr_message(&winner, None, true)
22535 .await
22536 .unwrap();
22537 let message = if accepted {
22538 first.expect("a passing rewrite never asks")
22539 } else {
22540 assert!(first.is_none());
22542 assert!(runner.state.parked);
22543 let store = ask::Questions::open();
22544 let asked: Vec<_> = store
22545 .list()
22546 .into_iter()
22547 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22548 .collect();
22549 assert_eq!(asked.len(), 1);
22550 assert!(!asked[0].detail.contains("secret"));
22551 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22552 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22553 assert_eq!(
22554 asked[0]
22555 .choices
22556 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22557 rewrite.is_some()
22558 );
22559 store
22562 .update(&asked[0].id, |q| {
22563 q.abandon("test");
22564 Ok(())
22565 })
22566 .unwrap();
22567 let again = runner
22568 .guarded_pr_message(&winner, None, true)
22569 .await
22570 .unwrap()
22571 .expect("an abandoned question falls back");
22572 let n = store
22573 .list()
22574 .into_iter()
22575 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22576 .count();
22577 assert_eq!(n, 1, "asked once per rejected text");
22578 again
22579 };
22580 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22581 assert!(!message.body.contains("secret"));
22582 assert!(!message.body.contains("prefix"));
22583 assert!(!message.title.contains("secret"));
22584 assert_eq!(
22585 message.body.contains("Add retries for failed requests."),
22586 accepted
22587 );
22588 assert_eq!(
22589 runner.state.seats["github-text-rewrite"].turns, 1,
22590 "only one rewrite invocation"
22591 );
22592 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22593 if accepted {
22594 assert!(runner.state.github_text.is_none());
22595 assert!(!runner.state.parked);
22596 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22597 assert!(message.body.contains("## Final verification"));
22598 assert!(message.body.contains("Evidence: final diff"));
22599 assert!(
22600 message
22601 .body
22602 .contains(&format!("magi:run/{}", runner.state.id))
22603 );
22604 }
22605 if !accepted {
22606 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22607 assert!(
22608 message
22609 .body
22610 .contains(&format!("magi:run/{}", runner.state.id))
22611 );
22612 }
22613 }
22614 }
22615
22616 fn scripted(id: &str, reply: &str) -> AgentSpec {
22617 let mut a = spec(id);
22618 a.command = vec![
22619 "sh".into(),
22620 "-c".into(),
22621 "printf '%s' \"$REWRITE_REPLY\"".into(),
22622 ];
22623 a.env.insert("REWRITE_REPLY".into(), reply.into());
22624 a
22625 }
22626
22627 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22628 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22629
22630 fn seed_public_summary(runner: &mut Runner) {
22632 let repo = &runner.state.repo;
22633 let g = |args: &[&str]| {
22634 let out = std::process::Command::new("git")
22635 .args(args)
22636 .current_dir(repo)
22637 .quiet()
22638 .output()
22639 .unwrap();
22640 assert!(out.status.success(), "git {args:?}");
22641 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22642 };
22643 g(&["init", "-q", "-b", "main"]);
22644 g(&[
22645 "-c",
22646 "user.name=Test",
22647 "-c",
22648 "user.email=test@example.invalid",
22649 "commit",
22650 "-q",
22651 "--allow-empty",
22652 "-m",
22653 "feat: fixture",
22654 ]);
22655 let head = g(&["rev-parse", "HEAD"]);
22656 g(&["update-ref", "refs/remotes/origin/main", &head]);
22657 let evidence = crate::pr_outcome::summary_evidence(
22658 &runner.state,
22659 crate::pr_outcome::FinalHead::At(&head),
22660 );
22661 let generated = &runner.state.candidates[0].summary;
22662 runner.state.change_summary = Some(crate::run::ChangeSummary {
22663 base: head.clone(),
22664 head,
22665 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22666 title: summary_title(generated).unwrap(),
22667 body: summary_without_title(generated),
22668 });
22669 }
22670
22671 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22672 let mut runner = runner_at(RunStatus::Gating);
22673 runner.state = state_with_summary("add retries", JA_SUMMARY);
22674 if review_only {
22675 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22676 }
22677 runner.state.repo = dir.to_owned();
22678 runner.state.candidates[0].worktree = dir.to_owned();
22679 runner.state.config.agents = agents;
22680 seed_public_summary(&mut runner);
22681 runner
22682 }
22683
22684 fn gate_questions(runner: &Runner) -> usize {
22685 ask::Questions::open()
22686 .list()
22687 .into_iter()
22688 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22689 .count()
22690 }
22691
22692 #[tokio::test]
22693 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22694 crate::run::pin_test_home();
22695 let dir = tempfile::tempdir().unwrap();
22696 let mut runner = rewrite_runner(
22697 dir.path(),
22698 true,
22699 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22700 );
22701 runner.state.config.roles.language_judge =
22702 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22703 let winner = runner.state.candidates[0].clone();
22704 let msg = runner
22705 .guarded_pr_message(&winner, None, true)
22706 .await
22707 .unwrap()
22708 .expect("an English rewrite posts without asking");
22709 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22710 assert!(msg.body.contains("<details>kept</details>"));
22711 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22712 assert_eq!(gate_questions(&runner), 0);
22713 assert!(runner.state.github_text.is_none());
22714 assert!(!runner.state.seats.contains_key("impl-A"));
22715 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22716 }
22717
22718 #[tokio::test]
22719 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22720 crate::run::pin_test_home();
22721 let dir = tempfile::tempdir().unwrap();
22722 let mut runner = rewrite_runner(
22723 dir.path(),
22724 false,
22725 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22726 );
22727 runner.state.config.roles.language_judge =
22728 Some(crate::config::AgentChoice::One("beta".into()));
22729 runner.state.seats.insert(
22730 "impl-A".into(),
22731 SeatState::new("impl-A", "alpha", runner.state.seed),
22732 );
22733 let winner = runner.state.candidates[0].clone();
22734 let msg = runner
22735 .guarded_pr_message(&winner, None, true)
22736 .await
22737 .unwrap()
22738 .expect("rewritten");
22739 assert!(msg.body.contains("<details>kept</details>"));
22740 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22741 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22742 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22743 }
22744
22745 #[tokio::test]
22746 async fn rewrite_reuses_a_resumable_implementer_session() {
22747 crate::run::pin_test_home();
22748 let dir = tempfile::tempdir().unwrap();
22749 let mut runner = rewrite_runner(
22750 dir.path(),
22751 false,
22752 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22753 );
22754 runner.state.config.roles.language_judge =
22755 Some(crate::config::AgentChoice::One("beta".into()));
22756 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22757 seat.turns = 1;
22758 runner.state.seats.insert("impl-A".into(), seat);
22759 let winner = runner.state.candidates[0].clone();
22760 runner
22761 .guarded_pr_message(&winner, None, true)
22762 .await
22763 .unwrap()
22764 .expect("rewritten");
22765 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22766 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22767 }
22768
22769 #[tokio::test]
22770 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22771 crate::run::pin_test_home();
22774 let dir = tempfile::tempdir().unwrap();
22775 let mut exhausted = spec("alpha");
22776 exhausted.command = vec!["false".into()];
22777 let mut runner = rewrite_runner(
22778 dir.path(),
22779 false,
22780 vec![exhausted, scripted("beta", EN_REWRITE)],
22781 );
22782 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22784 "alpha".into(),
22785 "beta".into(),
22786 ]));
22787 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22788 seat.turns = 1;
22789 runner.state.seats.insert("impl-A".into(), seat);
22790 let winner = runner.state.candidates[0].clone();
22791 let msg = runner
22792 .guarded_pr_message(&winner, None, true)
22793 .await
22794 .unwrap()
22795 .expect("the role rewrites after the implementer fails");
22796 assert!(msg.body.contains("`src/net.rs`"));
22797 assert_eq!(
22798 runner.state.seats["impl-A"].turns, 2,
22799 "the implementer was tried once"
22800 );
22801 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22802 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22803 }
22804
22805 #[tokio::test]
22806 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22807 crate::run::pin_test_home();
22808 let dir = tempfile::tempdir().unwrap();
22809 let mut broken = spec("bad");
22810 broken.command = vec!["false".into()];
22811 let mut runner = rewrite_runner(
22812 dir.path(),
22813 true,
22814 vec![
22815 broken,
22816 scripted("noformat", "Add retries."),
22817 scripted("good", EN_REWRITE),
22818 ],
22819 );
22820 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22821 "bad".into(),
22822 "noformat".into(),
22823 "good".into(),
22824 ]));
22825 let winner = runner.state.candidates[0].clone();
22826 let msg = runner
22827 .guarded_pr_message(&winner, None, true)
22828 .await
22829 .unwrap()
22830 .expect("the third agent rewrites");
22831 assert!(msg.body.contains("`src/net.rs`"));
22832 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22833 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22834 }
22835
22836 #[tokio::test]
22837 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22838 crate::run::pin_test_home();
22839 for (agents, role, saved) in [
22841 (
22842 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22843 Some("beta"),
22844 true,
22845 ),
22846 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22847 (Vec::new(), None, false),
22848 ] {
22849 let dir = tempfile::tempdir().unwrap();
22850 let mut runner = rewrite_runner(dir.path(), true, agents);
22851 runner.state.config.roles.language_judge =
22852 role.map(|r| crate::config::AgentChoice::One(r.into()));
22853 let winner = runner.state.candidates[0].clone();
22854 let first = runner
22855 .guarded_pr_message(&winner, None, true)
22856 .await
22857 .unwrap();
22858 assert!(first.is_none(), "withheld text parks the run");
22859 assert!(runner.state.parked);
22860 assert_eq!(gate_questions(&runner), 1);
22861 let g = runner.state.github_text.as_ref().unwrap();
22862 assert_eq!(g.rewrite.is_some(), saved);
22863 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22864 }
22865 }
22866
22867 #[tokio::test]
22868 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22869 crate::run::pin_test_home();
22870 let dir = tempfile::tempdir().unwrap();
22871 let mut runner = runner_at(RunStatus::Gating);
22872 runner.state =
22873 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22874 runner.state.repo = dir.path().to_owned();
22875 runner.state.candidates[0].worktree = dir.path().to_owned();
22876 let mut author = spec("alpha");
22877 author.command = vec![
22878 "sh".into(),
22879 "-c".into(),
22880 "printf '%s' \"$REWRITE_REPLY\"".into(),
22881 ];
22882 author.env.insert(
22883 "REWRITE_REPLY".into(),
22884 format!(
22885 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22886 "English sentence. ".repeat(600)
22887 ),
22888 );
22889 runner.state.config.agents = vec![author];
22890 seed_public_summary(&mut runner);
22891 let winner = runner.state.candidates[0].clone();
22892 assert!(
22893 runner
22894 .guarded_pr_message(&winner, None, true)
22895 .await
22896 .unwrap()
22897 .is_none()
22898 );
22899 let record = runner.state.github_text.clone().unwrap();
22900 let rewrite = record.rewrite.clone().unwrap();
22901 assert_eq!(rewrite.failed, ["body-language"]);
22902 assert!(
22903 rewrite
22904 .body
22905 .ends_with(&format!("magi:run/{}", runner.state.id))
22906 );
22907 let store = ask::Questions::open();
22908 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22909 assert!(
22910 q.choices
22911 .contains(&crate::github_text::USE_REWRITE.to_owned())
22912 );
22913 for label in [
22914 "ORIGINAL",
22915 "FALLBACK",
22916 "REWRITE",
22917 "Rules still failed: body-language",
22918 "truncated",
22919 ] {
22920 assert!(q.detail.contains(label), "{label}");
22921 }
22922 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22923 &runner.state,
22924 &crate::github_text::artifact_name(&record.fingerprint),
22925 ))
22926 .unwrap();
22927 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22928 runner.state =
22930 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22931 store
22932 .update(&q.id, |q| {
22933 q.say("まだ日本語")?;
22934 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22935 })
22936 .unwrap();
22937 assert!(
22938 runner
22939 .guarded_pr_message(&winner, None, true)
22940 .await
22941 .unwrap()
22942 .is_none()
22943 );
22944 let retry = runner.state.github_text.as_ref().unwrap();
22945 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22946 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22947 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22948 assert!(
22949 q.posting_comparison
22950 .as_ref()
22951 .unwrap()
22952 .matches_detail(&q.detail)
22953 );
22954 store
22955 .update(&q.id, |q| {
22956 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22957 })
22958 .unwrap();
22959 let message = runner
22960 .guarded_pr_message(&winner, None, true)
22961 .await
22962 .unwrap()
22963 .unwrap();
22964 assert_eq!(message.title, rewrite.title);
22965 assert_eq!(message.body, rewrite.body);
22966 assert_eq!(
22967 runner.state.github_text.as_ref().unwrap().choice,
22968 Some(GithubTextChoice::Rewrite)
22969 );
22970 let again = runner
22971 .guarded_pr_message(&winner, None, true)
22972 .await
22973 .unwrap()
22974 .unwrap();
22975 assert_eq!(again.title, message.title);
22976 assert_eq!(again.body, message.body);
22977 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22978 assert!(
22979 !runner
22980 .state
22981 .events
22982 .iter()
22983 .any(|e| e.message.contains(&rewrite.title)
22984 || e.message.contains("English sentence."))
22985 );
22986 }
22987
22988 #[tokio::test]
22989 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22990 crate::run::pin_test_home();
22991 let dir = tempfile::tempdir().unwrap();
22992 let mut runner = runner_at(RunStatus::Gating);
22993 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22994 runner.state.repo = dir.path().to_owned();
22995 runner.state.candidates[0].worktree = dir.path().to_owned();
22996 seed_public_summary(&mut runner);
22997 let winner = runner.state.candidates[0].clone();
22998 assert!(
22999 runner
23000 .guarded_pr_message(&winner, None, true)
23001 .await
23002 .unwrap()
23003 .is_none()
23004 );
23005 let record = runner.state.github_text.as_ref().unwrap();
23006 assert!(record.rewrite.is_none());
23007 let mut old = serde_json::to_value(record).unwrap();
23009 old.as_object_mut().unwrap().remove("rewrite");
23010 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23011 assert!(old.rewrite.is_none());
23012 let id = record.question.clone().unwrap();
23013 ask::Questions::open()
23014 .update(&id, |q| {
23015 q.choices.push(crate::github_text::USE_REWRITE.into());
23016 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23017 })
23018 .unwrap();
23019 let message = runner
23020 .guarded_pr_message(&winner, None, true)
23021 .await
23022 .unwrap()
23023 .unwrap();
23024 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23025 assert_eq!(
23026 runner.state.github_text.as_ref().unwrap().choice,
23027 Some(GithubTextChoice::Fallback)
23028 );
23029 }
23030
23031 #[tokio::test]
23032 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23033 crate::run::pin_test_home();
23034 let dir = tempfile::tempdir().unwrap();
23035 let mut runner = runner_at(RunStatus::Gating);
23036 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23037 runner.state.repo = dir.path().to_owned();
23038 runner.state.candidates[0].worktree = dir.path().to_owned();
23039 seed_public_summary(&mut runner);
23040 let winner = runner.state.candidates[0].clone();
23041 assert!(
23042 runner
23043 .guarded_pr_message(&winner, None, true)
23044 .await
23045 .unwrap()
23046 .is_none()
23047 );
23048 let store = ask::Questions::open();
23049 let find = |run: &str| {
23050 store
23051 .list()
23052 .into_iter()
23053 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23054 .unwrap()
23055 };
23056 let run = runner.state.id.clone();
23057 let q = find(&run);
23058 let fp = runner
23059 .state
23060 .github_text
23061 .as_ref()
23062 .unwrap()
23063 .fingerprint
23064 .clone();
23065 let name = crate::github_text::artifact_name(&fp);
23066 let path = crate::run::artifact_path(&runner.state, &name);
23067 assert!(q.detail.contains(&path.display().to_string()));
23068 assert!(q.detail.contains("title-language: title"));
23069 let saved = std::fs::read_to_string(&path).expect("artifact written");
23070 assert!(saved.contains("日本語のタイトル"));
23071 assert!(q.cwd.is_none());
23072 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23073 let say = |id: &str, text: &str, choice: &str| {
23075 store
23076 .update(id, |q| {
23077 q.thread.push(ask::Turn {
23078 who: ask::Who::Operator,
23079 body: text.to_owned(),
23080 at: jiff::Timestamp::now(),
23081 note: None,
23082 });
23083 q.answer(ask::Answer::Choice(choice.to_owned()))
23084 })
23085 .unwrap();
23086 };
23087 say(&q.id, "まだ日本語", "use my text");
23088 assert!(
23089 runner
23090 .guarded_pr_message(&winner, None, true)
23091 .await
23092 .unwrap()
23093 .is_none()
23094 );
23095 let second = find(&run);
23096 assert_ne!(second.id, q.id);
23097 assert!(second.detail.contains("did not pass"));
23098 assert!(second.detail.contains("日本語のタイトル"));
23101 assert!(!second.detail.contains("まだ日本語"));
23102 assert!(second.detail.contains(&path.display().to_string()));
23103 say(&second.id, "fix: retry failed requests", "use my text");
23104 let message = runner
23105 .guarded_pr_message(&winner, None, true)
23106 .await
23107 .unwrap()
23108 .expect("a vetted replacement is posted");
23109 assert_eq!(message.title, "fix: retry failed requests");
23110 assert_eq!(
23111 runner
23112 .state
23113 .github_text
23114 .as_ref()
23115 .unwrap()
23116 .chosen_title
23117 .as_deref(),
23118 Some("fix: retry failed requests")
23119 );
23120 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23121 let again = runner
23123 .guarded_pr_message(&winner, None, true)
23124 .await
23125 .unwrap()
23126 .unwrap();
23127 assert_eq!(again.title, message.title);
23128 assert_eq!(
23129 store
23130 .list()
23131 .into_iter()
23132 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23133 .count(),
23134 2
23135 );
23136 }
23137
23138 #[test]
23139 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23140 let mut state = state_with_summary(
23144 "add retries",
23145 "TITLE: fix(web): batch reads\n- reads run.json once",
23146 );
23147 state.config.graph.language = "ja".to_owned();
23148 let m = pr_message(&state, 'A');
23149 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23150 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23151
23152 let task = "今回やってほしいこと: results projector を直す";
23155 let mut state = state_with_summary(task, "- no title line");
23156 state.config.graph.language = "ja".to_owned();
23157 let m = pr_message(&state, 'A');
23158 assert_eq!(
23159 m.title,
23160 format!("chore: land candidate A of run {}", state.id)
23161 );
23162 assert!(
23163 m.body.contains(&format!(
23164 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23165 )),
23166 "{}",
23167 m.body
23168 );
23169 }
23170
23171 #[test]
23172 fn pr_message_scrubs_home_paths_and_addresses() {
23173 let state = state_with_summary(
23174 "fix it in /Users/someone/src/x",
23175 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23176 );
23177 let m = pr_message(&state, 'A');
23178 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23179 assert!(!m.body.contains(leak), "{}", m.body);
23180 }
23181 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23182 }
23183
23184 #[test]
23185 fn pr_message_survives_a_task_that_closes_details() {
23186 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23187 let m = pr_message(&state, 'A');
23188 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23189 }
23190
23191 #[test]
23192 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23193 let cmd = manual_merge_command(
23194 MergeStyle::Squash,
23195 Path::new("/repo"),
23196 "b",
23197 "fix: \"quoted\" $(x) `y`\n\nbody",
23198 );
23199 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23200 }
23201
23202 #[test]
23203 fn manual_merge_command_matches_the_configured_style() {
23204 let repo = Path::new("/repo");
23205 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23206
23207 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23208 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23209
23210 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23211 assert_eq!(
23212 squash,
23213 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23214 \"Merge magi run 0832 (candidate A)\""
23215 );
23216
23217 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23218 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23219 }
23220
23221 #[test]
23222 fn a_nudge_gets_a_quarter_of_the_budget() {
23223 assert_eq!(retry_budget(secs(1200), true), secs(300));
23225 assert_eq!(retry_budget(secs(3600), true), secs(900));
23226 }
23227
23228 #[test]
23229 fn a_resent_prompt_keeps_the_whole_budget() {
23230 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23233 assert_eq!(retry_budget(secs(60), false), secs(60));
23234 }
23235
23236 #[test]
23237 fn the_floor_never_exceeds_the_original_budget() {
23238 assert_eq!(retry_budget(secs(60), true), secs(60));
23242 assert_eq!(retry_budget(secs(480), true), secs(120));
23243 assert_eq!(retry_budget(secs(0), true), secs(0));
23244 }
23245
23246 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23247 agent::CommandEvidence {
23248 id: "item1".to_owned(),
23249 description: "cargo test".to_owned(),
23250 exit_code,
23251 result_summary: String::new(),
23252 source: "codex".to_owned(),
23253 }
23254 }
23255
23256 #[test]
23257 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23258 assert!(!has_unconfirmed_command(&[]));
23262 }
23263
23264 #[test]
23265 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23266 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23270 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23271 assert!(!has_unconfirmed_command(&[
23272 evidence(Some(0)),
23273 evidence(Some(101))
23274 ]));
23275 }
23276
23277 #[test]
23278 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23279 assert!(has_unconfirmed_command(&[
23280 evidence(Some(0)),
23281 evidence(None)
23282 ]));
23283 }
23284
23285 #[test]
23286 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23287 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23288 assert_eq!(
23289 verified_noop_claim(true, &[], text).as_deref(),
23290 Some("already fixed by b32cfc4, on main.")
23291 );
23292 }
23293
23294 #[test]
23295 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23296 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23299 assert!(verified_noop_claim(false, &[], text).is_none());
23300 }
23301
23302 #[test]
23303 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23304 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23305 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23306 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23308 }
23309
23310 #[test]
23311 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23312 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23313 }
23314
23315 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23318 runner.state.candidates = shape
23319 .iter()
23320 .enumerate()
23321 .map(|(i, &(empty, verified))| Candidate {
23322 index: i,
23323 label: (b'A' + i as u8) as char,
23324 agent: "sonnet".to_owned(),
23325 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23326 worktree: PathBuf::from(format!("/wt/{i}")),
23327 summary: String::new(),
23328 stat: String::new(),
23329 files: 0,
23330 commits: 0,
23331 empty,
23332 failed: None,
23333 verified_noop: verified.map(str::to_owned),
23334 duration_ms: 0,
23335 folded: false,
23336 })
23337 .collect();
23338 }
23339
23340 #[test]
23341 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23342 ask_test_home();
23343 let mut runner = runner_at(RunStatus::Implementing);
23344 set_candidates(
23345 &mut runner,
23346 &[
23347 (true, Some("already on main at b32cfc4")),
23348 (true, Some("same fix, see the existing test")),
23349 ],
23350 );
23351
23352 runner
23353 .after_implement()
23354 .expect("a verified no-op is not an error");
23355
23356 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23357 }
23358
23359 #[test]
23360 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23361 ask_test_home();
23362 let mut runner = runner_at(RunStatus::Implementing);
23363 set_candidates(
23367 &mut runner,
23368 &[(true, Some("already on main at b32cfc4")), (true, None)],
23369 );
23370
23371 let err = runner
23372 .after_implement()
23373 .expect_err("an unverified empty candidate must still fail the run");
23374
23375 assert!(
23376 err.to_string().contains("no candidate produced a change"),
23377 "{err}"
23378 );
23379 assert_eq!(runner.state.status, RunStatus::Failed);
23380 }
23381
23382 #[test]
23383 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23384 ask_test_home();
23385 let mut runner = runner_at(RunStatus::Implementing);
23386 set_candidates(&mut runner, &[(true, None), (true, None)]);
23387
23388 let err = runner
23389 .after_implement()
23390 .expect_err("no candidate declared anything; this is an ordinary failure");
23391
23392 assert!(
23393 err.to_string().contains("no candidate produced a change"),
23394 "{err}"
23395 );
23396 assert_eq!(runner.state.status, RunStatus::Failed);
23397 }
23398}
23399
23400#[derive(Debug)]
23403struct GraftRestoreFailed(String);
23404
23405impl std::fmt::Display for GraftRestoreFailed {
23406 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23407 write!(
23408 f,
23409 "could not restore the winner to its pre-graft commit: {}",
23410 self.0
23411 )
23412 }
23413}
23414
23415impl std::error::Error for GraftRestoreFailed {}
23416
23417#[cfg(test)]
23421mod resource_wait_tests {
23422 use super::*;
23423 use std::collections::BTreeMap;
23424
23425 fn sh(id: &str, script: &str) -> AgentSpec {
23426 AgentSpec {
23427 id: id.to_owned(),
23428 kind: crate::config::AgentKind::Command,
23429 model: None,
23430 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23431 extra_args: Vec::new(),
23432 env: BTreeMap::new(),
23433 prompt_delivery: None,
23434 max_turn_secs: None,
23435 }
23436 }
23437
23438 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23439 SeatJob {
23440 seat: SeatState::new("impl-A", &spec.id, 1),
23441 spec,
23442 cwd: dir.to_path_buf(),
23443 prompt: "do it".to_owned(),
23444 timeout,
23445 allow_write: true,
23446 sessions: false,
23447 artifacts: dir.join("art"),
23448 stem: "impl-A".to_owned(),
23449 handover: None,
23450 }
23451 }
23452
23453 fn state(resource_wait_secs: u64) -> RunState {
23454 let mut config = Config::default();
23455 config.graph.timeout_resource_wait = resource_wait_secs;
23456 RunState::new(
23457 PathBuf::from("/repo"),
23458 "main".to_owned(),
23459 "abc1234".to_owned(),
23460 "task".to_owned(),
23461 config,
23462 )
23463 }
23464
23465 fn hold(cache: &Path) -> crate::cache::Guard {
23466 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23467 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23468 crate::cache::AcquireOutcome::Acquired(g) => g,
23469 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23470 }
23471 }
23472
23473 async fn run(
23474 jobs: Vec<SeatJob>,
23475 sem: Arc<Semaphore>,
23476 cache: Option<&Path>,
23477 st: &mut RunState,
23478 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23479 let prompts = Prompts::default();
23480 let ctx = WaveCtx {
23481 run: "r",
23482 node: "implement",
23483 prompts: &prompts,
23484 cache,
23485 round: None,
23486 carry_seats: false,
23487 pauses: None,
23488 };
23489 wave(jobs, sem, &ctx, st, 0).await
23490 }
23491
23492 #[tokio::test]
23493 async fn verification_cache_wait_preserves_the_command_budget() {
23494 crate::run::pin_test_home();
23495 let tmp = tempfile::tempdir().unwrap();
23496 let cache = tmp.path().join("target");
23497 let guard = hold(&cache);
23498 let release = tokio::spawn(async move {
23499 tokio::time::sleep(Duration::from_millis(100)).await;
23500 guard.release();
23501 });
23502 let mut st = state(10);
23503 let budget = Duration::from_millis(50);
23504 let wait = repository_wait(&st, None);
23505 let (outcomes, _) = with_cache_lease(
23506 &mut st,
23507 Some(&cache),
23508 "gate",
23509 "gate",
23510 tmp.path(),
23511 "abc1234",
23512 budget,
23513 "test gate",
23514 &wait,
23515 |_, available| async move {
23516 assert_eq!(
23517 available, budget,
23518 "contention must not shorten verification"
23519 );
23520 (Vec::new(), false, Vec::new())
23521 },
23522 )
23523 .await
23524 .unwrap();
23525 release.await.unwrap();
23526 assert!(outcomes.is_empty(), "{outcomes:?}");
23527 assert!(
23528 st.events
23529 .iter()
23530 .any(|event| event.message.contains("waiting for"))
23531 );
23532 assert!(st.resource_wait.is_none());
23533 hold(&cache).release();
23535 }
23536
23537 #[tokio::test]
23538 async fn verification_cache_wait_expires_without_starting_commands() {
23539 crate::run::pin_test_home();
23540 let tmp = tempfile::tempdir().unwrap();
23541 let cache = tmp.path().join("target");
23542 let _held = hold(&cache);
23543 let mut st = state(0);
23544 let wait = repository_wait(&st, None);
23545 let (outcomes, _) = with_cache_lease(
23546 &mut st,
23547 Some(&cache),
23548 "gate",
23549 "gate",
23550 tmp.path(),
23551 "abc1234",
23552 Duration::from_secs(60),
23553 "test gate",
23554 &wait,
23555 |_, _| async { panic!("commands must not start without the cache lease") },
23556 )
23557 .await
23558 .unwrap();
23559 assert_eq!(outcomes.len(), 1);
23560 assert!(outcomes[0].resource_blocked);
23561 assert!(st.resource_wait.is_none());
23562 assert!(matches!(
23563 st.resource_block.unwrap().cause,
23564 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23565 ));
23566 }
23567
23568 #[tokio::test]
23569 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23570 crate::run::pin_test_home();
23571 let tmp = tempfile::tempdir().unwrap();
23572 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23573 let cache = tmp.path().join("cache");
23574 let mut state = state(21600);
23575 let wait = repository_wait(&state, None);
23576 let (outcomes, _) = with_cache_lease(
23577 &mut state,
23578 Some(&cache),
23579 "gate",
23580 "gate",
23581 tmp.path(),
23582 "head",
23583 Duration::from_secs(60),
23584 "gate",
23585 &wait,
23586 |_, _| async { panic!("freshness failure must not start verification") },
23587 )
23588 .await
23589 .unwrap();
23590 assert_eq!(outcomes.len(), 1);
23591 assert!(outcomes[0].resource_blocked);
23592 let block = state.resource_block.unwrap();
23593 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23594 assert!(!block.description().contains("timed out"));
23595 }
23596
23597 #[tokio::test]
23598 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23599 crate::run::pin_test_home();
23600 let tmp = tempfile::tempdir().unwrap();
23601 let cache = tmp.path().join("cache");
23602 let _held = hold(&cache);
23603 let mut state = state(21600);
23604 let pause = Pause::new();
23605 let interrupt = Pause::new();
23606 let cause = ParkCause::Operator;
23607 pause.park_for(cause.clone());
23608 let prompts = Prompts::default();
23609 let ctx = WaveCtx {
23610 run: "test",
23611 node: "fix",
23612 prompts: &prompts,
23613 cache: Some(&cache),
23614 round: None,
23615 carry_seats: false,
23616 pauses: Some((&pause, &interrupt)),
23617 };
23618 let result = wave(
23619 vec![job(
23620 sh("writer", "exit 99"),
23621 tmp.path(),
23622 Duration::from_secs(1),
23623 )],
23624 Arc::new(Semaphore::new(1)),
23625 &ctx,
23626 &mut state,
23627 0,
23628 )
23629 .await;
23630 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23631 assert_eq!(result[0].1.turns, 0);
23632 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23633 assert_eq!(
23634 state.resource_block.unwrap().cause,
23635 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23636 );
23637 assert!(state.active.is_empty());
23638 }
23639
23640 #[tokio::test]
23641 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23642 crate::run::pin_test_home();
23643 let tmp = tempfile::tempdir().unwrap();
23644 let cache = tmp.path().join("target-a");
23645 let guard = hold(&cache);
23646 tokio::spawn(async move {
23648 tokio::time::sleep(Duration::from_millis(1500)).await;
23649 guard.release();
23650 });
23651 let mut st = state(60);
23652 let results = run(
23653 vec![job(
23654 sh("alpha", "echo ok"),
23655 tmp.path(),
23656 Duration::from_secs(1),
23657 )],
23658 Arc::new(Semaphore::new(1)),
23659 Some(&cache),
23660 &mut st,
23661 )
23662 .await;
23663 assert!(
23664 matches!(results[0].2, AgentOutcome::Ok(_)),
23665 "the wait must not be charged to the agent"
23666 );
23667 assert!(st.resource_block.is_none());
23668 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23669 assert!(st.resource_wait.is_none() && st.active.is_empty());
23670 }
23671
23672 #[tokio::test]
23673 #[cfg(unix)]
23674 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23675 use std::os::fd::AsRawFd as _;
23676 crate::run::pin_test_home();
23677 let tmp = tempfile::tempdir().unwrap();
23678 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23679 let common = tmp.path().join(".git");
23680 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23681 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23682 assert_eq!(
23683 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23684 0
23685 );
23686 let mut st = state(1);
23687 st.config.coordination = crate::config::test_coordination();
23688 let mut reader = job(
23689 sh("reader", "echo must-not-run"),
23690 tmp.path(),
23691 Duration::from_secs(30),
23692 );
23693 reader.allow_write = false;
23694 let results = run(
23695 vec![reader.clone()],
23696 Arc::new(Semaphore::new(1)),
23697 None,
23698 &mut st,
23699 )
23700 .await;
23701 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23702 assert_eq!(results[0].1.turns, 0);
23703 assert!(FailClass::of(&results[0].2).is_none());
23704 assert_eq!(
23705 st.resource_block.as_ref().unwrap().kind,
23706 "repository coordination"
23707 );
23708 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23709 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23710 drop(external);
23711 let editor = crate::repo_lock::SeatLease::for_checkout(
23712 tmp.path(),
23713 true,
23714 crate::config::test_coordination().as_ref(),
23715 )
23716 .await
23717 .unwrap();
23718 let pause = Pause::new();
23719 let interrupt = Pause::new();
23720 let prompts = Prompts::default();
23721 let ctx = WaveCtx {
23722 run: "r",
23723 node: "review",
23724 prompts: &prompts,
23725 cache: None,
23726 round: None,
23727 carry_seats: false,
23728 pauses: Some((&pause, &interrupt)),
23729 };
23730 let (results, ()) = tokio::join!(
23731 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23732 async {
23733 tokio::time::sleep(Duration::from_millis(50)).await;
23734 pause.park();
23735 }
23736 );
23737 assert!(
23738 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23739 );
23740 assert_eq!(results[0].1.turns, 0);
23741 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23742 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23743 drop(editor);
23744 }
23745
23746 #[tokio::test]
23747 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23748 crate::run::pin_test_home();
23749 let tmp = tempfile::tempdir().unwrap();
23750 let cache = tmp.path().join("target-b");
23751 let _held = hold(&cache);
23752 let mut st = state(1);
23753 let results = run(
23754 vec![job(
23755 sh("alpha", "echo ok"),
23756 tmp.path(),
23757 Duration::from_secs(1),
23758 )],
23759 Arc::new(Semaphore::new(1)),
23760 Some(&cache),
23761 &mut st,
23762 )
23763 .await;
23764 let (_, seat, out) = &results[0];
23765 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23766 assert!(wave_blocked(&results));
23767 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23768 assert_eq!(seat.turns, 0, "nothing was asked");
23769 assert!(
23770 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23771 "no agent was spawned"
23772 );
23773 assert!(FailClass::of(out).is_none(), "not a failure class");
23774 let block = st.resource_block.as_ref().expect("recorded for the resume");
23775 assert_eq!(block.kind, "shared build cache");
23776 assert!(matches!(
23777 block.cause,
23778 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23779 ));
23780 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23781 assert!(st.active.is_empty());
23782 }
23783
23784 #[tokio::test]
23785 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23786 crate::run::pin_test_home();
23787 let tmp = tempfile::tempdir().unwrap();
23788 let mut st = state(1);
23789 let results = run(
23790 vec![job(
23791 sh("alpha", "echo ok"),
23792 tmp.path(),
23793 Duration::from_secs(1),
23794 )],
23795 Arc::new(Semaphore::new(0)),
23796 None,
23797 &mut st,
23798 )
23799 .await;
23800 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23801 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23802 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23803 }
23804
23805 #[tokio::test]
23806 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23807 crate::run::pin_test_home();
23808 let tmp = tempfile::tempdir().unwrap();
23809 let mut st = state(60);
23810 let results = run(
23811 vec![job(
23812 sh("alpha", "sleep 30"),
23813 tmp.path(),
23814 Duration::from_secs(1),
23815 )],
23816 Arc::new(Semaphore::new(1)),
23817 None,
23818 &mut st,
23819 )
23820 .await;
23821 let out = &results[0].2;
23822 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23823 let class = FailClass::of(out).expect("a failure");
23824 assert_eq!(class, FailClass::Timeout);
23825 assert!(should_hand_over(None, &class), "existing handover policy");
23826 assert!(st.resource_block.is_none());
23827 }
23828}