1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533pub(crate) struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
775 state.set_origin(origin)?;
776 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777 state.status = RunStatus::Prep;
778 state.event("start", format!("run {} created", state.id));
779 startup.finish(&mut state, Ok(()))?;
780 Ok(Self {
781 state,
782 roles,
783 sem: Arc::new(Semaphore::new(max_parallel)),
784 pause: Pause::new(),
785 interrupt: Pause::new(),
786 })
787 }
788
789 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803 Self::review_taking_over(repo, branch, config, None, origin).await
804 }
805
806 pub async fn review_taking_over(
813 repo: &Path,
814 branch: &str,
815 config: Config,
816 takeover: Option<crate::handover::Takeover>,
817 origin: Origin,
818 ) -> Result<Self> {
819 let repo = git::toplevel(repo).await?;
820 let missing = agent::missing_programs(&config.agents);
821 if !missing.is_empty() {
822 bail!(
823 "these agent programs are not on PATH: {}. Fix the roster in \
824 magi.toml or install them.",
825 missing.join(", ")
826 );
827 }
828 let base_branch =
829 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830 .await?;
831 if base_branch == branch {
832 bail!("`{branch}` is the base branch; there is nothing to review against");
833 }
834 let fetched_base = resolve_base(
835 &repo,
836 &base_branch,
837 &config.merge.remote,
838 config.coordination.as_ref(),
839 )
840 .await?;
841 let base_commit = fetched_base.commit.clone();
842
843 let roles = config.resolve_roles()?;
844 let max_parallel = config.graph.max_parallel.max(1);
845 let mut state = RunState::new(
846 repo.clone(),
847 base_branch,
848 base_commit.clone(),
849 format!("Review the work already on branch `{branch}`."),
850 config,
851 );
852 state.fetched_base = Some(fetched_base);
853 state.set_origin(origin)?;
854 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856 let opened = async {
857 startup.stage(&mut state, "handover")?;
858
859 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866 earlier: Vec::new(),
867 home: crate::run::home(),
868 choice: None,
869 task_instruction: None,
870 });
871 let wait = repository_wait(&state, None);
872 let coordination_config = state.config.coordination.clone();
873 let coordination =
874 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875 .await?;
876 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877 if let Some(released) = &released {
878 state.event(
879 "release",
880 format!(
881 "took `{branch}` over from run {}: its worktree was released: {}",
882 crate::run::short_of(&released.old_id),
883 released.audit
884 ),
885 );
886 }
887 let opened = async {
888 startup.stage(&mut state, "review-preflight")?;
889 state.foreign_checkout = crate::handover::preflight_review(
893 &repo,
894 branch,
895 &takeover.home,
896 &state.config.merge.remote,
897 )
898 .await?;
899 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901 }
902 if let Some(choice) = takeover.choice.as_ref()
903 && let Err(e) =
904 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905 .await
906 {
907 return Err(e.context("applying the owner's answer about the diverged branch"));
908 }
909 for id in takeover.earlier.iter().rev() {
915 if let Ok(old) = RunState::load_under(id, &takeover.home)
916 && let Some(head) = old.pr_head.clone()
917 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918 {
919 state.pr_head = Some(head);
920 break;
921 }
922 }
923 drop(coordination);
924 startup.stage(&mut state, "open-review")?;
925 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926 }
927 .await;
928 if opened.is_err()
929 && let Some(released) = &released
930 {
931 match crate::repo_lock::SeatLease::operation_with(
934 &repo,
935 coordination_config.as_ref(),
936 &wait,
937 )
938 .await
939 {
940 Ok(_coordination) => {
941 released.restore(&repo, branch).await;
942 let outcome = RunState::load_under(&released.old_id, &takeover.home)
943 .map(|old| {
944 format!(
945 "predecessor released_to={:?}; candidate path exists={}",
946 old.released_to,
947 old.candidates.iter().any(|c| {
948 c.branch == branch && c.worktree.exists()
949 }),
950 )
951 })
952 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954 }
955 Err(error) => {
956 tracing::warn!(
957 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958 state.id
959 );
960 state.event(
961 "startup-restore",
962 format!("restoration not attempted: {error:#}"),
963 );
964 }
965 }
966 }
967 opened
968 }
969 .await;
970 startup.finish(&mut state, opened)?;
971 Ok(Self {
972 state,
973 roles,
974 sem: Arc::new(Semaphore::new(max_parallel)),
975 pause: Pause::new(),
976 interrupt: Pause::new(),
977 })
978 }
979
980 pub async fn recover_pr(
982 repo: &Path,
983 task: &str,
984 selection: crate::pr_recovery::Selection,
985 mut config: Config,
986 ) -> Result<Self> {
987 selection.configure(&mut config)?;
988 let assignment = crate::run::task_assignment_for(task)?;
989 selection.check_task_assignment(task, &selection.instruction, assignment)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.set_origin(crate::run::Origin::queue(task))?;
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.execution_runs().last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.ensure_base_episode();
1417 state.record_base_sync_stop(&head);
1418 state.save()?;
1419 let fetched =
1420 git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421 anyhow::ensure!(
1422 fetched.ok(),
1423 "cannot authorize base recovery without a fetched base"
1424 );
1425 let approved_tip = git::rev_parse(
1426 &state.repo,
1427 &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428 )
1429 .await?;
1430 if resumption {
1431 anyhow::ensure!(
1432 crate::daemon::compatible_run_instruction(&state, &queued),
1433 "resumption instruction does not match the retained run"
1434 );
1435 let approval = queued
1436 .resumption
1437 .as_ref()
1438 .context("no operator resumption")?;
1439 anyhow::ensure!(
1440 approval.instruction == queued.instruction,
1441 "resumption approval does not match the current task instruction"
1442 );
1443 state.authorize_base_sync_resumption(task, approval)?;
1444 } else {
1445 state.authorize_base_sync(task)?;
1446 }
1447 let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448 state
1449 .base_sync_recovery
1450 .authorizations
1451 .last_mut()
1452 .unwrap()
1453 .scope = Some(crate::base_sync_recovery::Scope {
1454 tip: approved_tip,
1455 instruction: queued.instruction.clone(),
1456 episode,
1457 });
1458 state.event(
1459 "base-recovery",
1460 "owner authorized a bounded base synchronization interval; fresh verification required",
1461 );
1462 state.save()?;
1463 Ok(())
1464 }
1465
1466 pub fn resume(id: &str) -> Result<Self> {
1468 let state = RunState::load(id)?;
1469 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1470 if state.startup.as_ref().is_some_and(|s| !s.complete)
1471 || (state.status == RunStatus::Prep
1472 && state.instruction.trim().is_empty()
1473 && state.candidates.is_empty()
1474 && state.prep_complete == Some(false))
1475 {
1476 bail!(
1477 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1478 );
1479 }
1480 crate::pr_head::ensure_owner(&state)?;
1481 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1482 bail!(
1483 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1484 );
1485 }
1486 if let Some(to) = &state.released_to {
1487 bail!(
1488 "run {} cannot be resumed: its worktree was released to run {}",
1489 state.short(),
1490 crate::run::short_of(to)
1491 );
1492 }
1493 let roles = state.config.resolve_roles()?;
1494 if let Some(id) = &state.config.graph.selected_implementer
1495 && state.candidates.iter().any(|c| c.agent != *id)
1496 {
1497 bail!(
1498 "replacement implementer pin does not match the saved candidates; task must stay held"
1499 );
1500 }
1501 let max_parallel = state.config.graph.max_parallel.max(1);
1502 Ok(Self {
1503 state,
1504 roles,
1505 sem: Arc::new(Semaphore::new(max_parallel)),
1506 pause: Pause::new(),
1507 interrupt: Pause::new(),
1508 })
1509 }
1510
1511 pub async fn execute(&mut self) -> Result<()> {
1518 anyhow::ensure!(
1519 self.state.startup.as_ref().is_none_or(|s| s.complete),
1520 "startup incomplete; inspect retained evidence before explicit recovery"
1521 );
1522 if let Some(selection) = &self.state.pr_recovery
1523 && (selection.source_proof.is_some() || selection.source_run.is_some())
1524 {
1525 Box::pin(selection.verify_source(&self.state.repo)).await?;
1526 }
1527 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1530 || self.state.land_approval.is_some()
1531 || self.state.merge_decision.is_some()
1532 || !self.state.merge_repairs.is_empty()
1533 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1534 {
1535 let claim = FixClaim::acquire(&self.state.dir())?;
1536 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1537 let disk = RunState::load(&self.state.id)?;
1538 let checkpoint = |s: &RunState| {
1539 (
1540 s.merge_repairs.len(),
1541 s.merge_repairs
1542 .last()
1543 .map(|r| (r.question.clone(), r.phase)),
1544 s.land_approval
1545 .as_ref()
1546 .map(|a| (a.question.clone(), a.head.clone())),
1547 )
1548 };
1549 if checkpoint(&disk) != checkpoint(&self.state) {
1550 bail!(
1551 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1552 );
1553 }
1554 Some(claim)
1555 } else {
1556 None
1557 };
1558 let mut execution = self.claim_driver().await?;
1559 let runner = &mut *execution.runner;
1560 let wait = runner.repository_wait();
1561 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1564 let result = match result {
1565 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1566 record_repository_block(&mut runner.state, &e);
1567 runner.park_here()?;
1568 Ok(())
1569 }
1570 other => other,
1571 };
1572 execution.driver.finish(&mut runner.state).await?;
1573 let ended = if result.is_err() {
1574 Some({
1575 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1576 if let Err(error) = &result {
1577 notice.message = format!(
1578 "Run {} stopped with an error: {}",
1579 runner.state.id,
1580 error.to_string().lines().next().unwrap_or("unknown error")
1581 );
1582 }
1583 notice
1584 })
1585 } else {
1586 crate::notices::run_ended(&runner.state)
1587 };
1588 if let Some(notice) = ended {
1589 crate::notices::raise(notice);
1590 }
1591 result
1592 }
1593
1594 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1595 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1596 }
1597
1598 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1599 let wait = self.repository_wait();
1600 coordinate_operation(&mut self.state, cwd, &wait).await
1601 }
1602
1603 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1605 let owner = match crate::pr_head::owner_lock_wait(
1606 &crate::run::home(),
1607 &self.state.id,
1608 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1609 )
1610 .await
1611 {
1612 Ok(owner) => owner,
1613 Err(error) => {
1614 let detail = format!("worktree maintenance or ownership update: {error:#}");
1615 self.state.resource_block = Some(crate::run::ResourceBlock {
1619 cause: crate::run::ResourceBlockCause::Other,
1620 node: "start".into(),
1621 kind: "worktree maintenance".into(),
1622 detail: detail.clone(),
1623 at: Timestamp::now(),
1624 });
1625 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1626 }
1627 };
1628 let disk = RunState::load(&self.state.id)?;
1629 crate::pr_head::ensure_owner(&disk)?;
1630 let claimed =
1631 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1632 anyhow::ensure!(
1637 disk.liveness(claimed) != crate::run::Liveness::Live,
1638 "run already has a live driver; wait for its execution to end"
1639 );
1640 if let Some(to) = &disk.released_to {
1641 bail!(
1642 "run {} cannot continue: its worktree was released to run {}",
1643 self.state.short(),
1644 crate::run::short_of(to)
1645 );
1646 }
1647 anyhow::ensure!(
1648 serde_json::to_value(&disk.base_sync_recovery)?
1649 == serde_json::to_value(&self.state.base_sync_recovery)?,
1650 "base recovery changed since this run was loaded; reload the saved run"
1651 );
1652 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1653 let mut execution = DriverExecution {
1654 runner: self,
1655 driver,
1656 };
1657 let _owner = owner;
1660 crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1661 .await?;
1662 let state = &mut execution.runner.state;
1663 state.parked = false;
1668 state.resource_wait = None;
1669 state.resource_block = None;
1670 state.clear_active();
1677 let pid = std::process::id();
1692 state.driver_pid = Some(pid);
1693 state.driver_started_at = crate::proc::process_started_at(pid);
1694 state.driver_exited = false;
1695 state.driver_claimed = Some(true);
1696 state.begin_pass();
1697 execution.driver.bind(state);
1698 state.save()?;
1699 execution.driver.record(state)?;
1700 Ok(execution)
1701 }
1702
1703 async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1706 let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1707 if let Err(error) = &result
1708 && !error.is::<crate::repo_lock::ResourceBlocked>()
1709 && self
1710 .state
1711 .repair_grants
1712 .last()
1713 .is_some_and(|g| g.conflict.is_none())
1714 {
1715 self.state.repair_grants.last_mut().unwrap().phase =
1716 crate::repair_budget::Phase::Complete;
1717 self.state.status = RunStatus::Blocked;
1718 self.state.save()?;
1719 bail!(
1720 "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1721 self.state.passes.last().map_or(0, |p| p.index)
1722 );
1723 }
1724 result
1725 }
1726
1727 async fn execute_granted_repair(&mut self) -> Result<()> {
1730 use crate::repair_budget::{Kind, Phase};
1731 let index = self.state.repair_grants.len() - 1;
1732 let winner = self
1733 .state
1734 .winner()
1735 .cloned()
1736 .context("repair winner missing")?;
1737 loop {
1738 let grant = self.state.repair_grants[index].clone();
1739 match grant.phase {
1740 Phase::Pending => {
1741 if grant.spent >= grant.receipt.request.rounds {
1742 self.state.repair_grants[index].phase = Phase::Complete;
1743 self.state.status = RunStatus::Blocked;
1744 self.state.save()?;
1745 return Ok(());
1746 }
1747 self.state.repair_grants[index].spent += 1;
1750 self.state.repair_grants[index].phase = Phase::Fixing;
1751 self.state.save()?;
1752 match grant.receipt.request.kind {
1753 Kind::BaseConflict => self.repair_base_conflict(true).await?,
1754 Kind::Gate => {
1755 let failed = self.state.gate.clone();
1756 let _ = self.gate_fix_round(&winner, &failed).await?;
1757 }
1758 Kind::Review => {
1759 let mut record = self
1760 .state
1761 .reviews
1762 .last()
1763 .cloned()
1764 .context("review failure missing")?;
1765 let round = record.round;
1766 let step = self.run_fix_round(&winner, &mut record).await?;
1767 self.state.reviews[round - 1] = record;
1768 if matches!(step, FixStep::Recorded { report_lost: true }) {
1769 self.state.repair_grants[index].phase = Phase::Complete;
1770 self.state.status = RunStatus::Blocked;
1771 self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1772 self.state.save()?;
1773 return Ok(());
1774 }
1775 if matches!(step, FixStep::Resource) {
1776 self.stop_for_resource()?;
1777 return Ok(());
1778 }
1779 self.state.save()?;
1780 }
1781 }
1782 if self.resource_halt()? {
1783 return Ok(());
1784 }
1785 }
1786 Phase::Fixing => {
1787 if grant.receipt.request.kind == Kind::BaseConflict {
1788 self.repair_base_conflict(false).await?;
1789 }
1790 let _coordination = self.coordinate(&winner.worktree).await?;
1793 let rescued = git::rescue_commit(
1794 &winner.worktree,
1795 "magi: preserve interrupted granted repair",
1796 )
1797 .await?;
1798 self.state.note_withheld("repair-grant", &rescued.withheld);
1799 self.state
1802 .verification_history
1803 .push(crate::run::VerificationEvidence {
1804 superseded_by: None,
1805 archived_at: Some(Timestamp::now()),
1806 fixes_spent: Some(self.state.gate_fixes.len()),
1807 gate_head: self.state.gate_head.take(),
1808 gate_ran: std::mem::take(&mut self.state.gate_ran),
1809 gate: std::mem::take(&mut self.state.gate),
1810 e2e: self.state.head_e2e.take(),
1811 });
1812 if let Some(e2e) = self.state.gate_repair_e2e.take() {
1813 self.state
1814 .verification_history
1815 .push(crate::run::VerificationEvidence {
1816 superseded_by: None,
1817 archived_at: Some(Timestamp::now()),
1818 fixes_spent: Some(self.state.gate_fixes.len()),
1819 gate_head: None,
1820 gate_ran: false,
1821 gate: Vec::new(),
1822 e2e: Some(e2e),
1823 });
1824 }
1825 self.state.repair_grants[index].review_start = self.state.reviews.len();
1826 self.state.repair_grants[index].head =
1827 git::rev_parse(&winner.worktree, "HEAD").await?;
1828 self.state.repair_grants[index].phase = Phase::Reviewing;
1829 if grant.receipt.request.kind == Kind::BaseConflict {
1830 self.state.base_sync_recovery.stop = None;
1831 if let Some(sync) = &mut self.state.base_sync {
1832 sync.tip = grant
1833 .receipt
1834 .request
1835 .tip
1836 .clone()
1837 .context("repair base tip missing")?;
1838 sync.conflict = None;
1839 sync.behind = 0;
1840 }
1841 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1842 }
1843 self.state.save()?;
1844 }
1845 Phase::Reviewing => {
1846 self.review_loop_pass().await?;
1847 if self.resource_halt()? {
1848 return Ok(());
1849 }
1850 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1851 let clean = self.state.reviews.len() > grant.review_start
1852 && self.state.reviews.last().is_some_and(|r| {
1853 r.head == head
1854 && r.clean
1855 && r.expected > 0
1856 && r.answered == r.expected
1857 && matches!(
1858 r.e2e_status(),
1859 E2eStatus::Passed | E2eStatus::NotConfigured
1860 )
1861 });
1862 if !clean {
1863 self.state.repair_grants[index].phase = if grant.receipt.request.kind
1865 == Kind::Review
1866 && grant.spent < grant.receipt.request.rounds
1867 && self.state.reviews.last().is_some_and(|r| {
1868 r.fix.is_none()
1869 && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1870 }) {
1871 Phase::Pending
1872 } else {
1873 Phase::Complete
1874 };
1875 self.state.status = RunStatus::Blocked;
1876 self.state.event(
1877 "repair-grant",
1878 "fresh review did not clear the repaired head; no publication",
1879 );
1880 self.state.save()?;
1881 if self.state.repair_grants[index].phase == Phase::Pending {
1882 continue;
1883 }
1884 return Ok(());
1885 }
1886 self.state.repair_grants[index].phase = Phase::Gating;
1887 self.state.save()?;
1888 }
1889 Phase::Gating => {
1890 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1891 self.state.reconcile_verification_head(&head);
1892 let wait = self.repository_wait();
1893 if self
1894 .state
1895 .reviews
1896 .last()
1897 .is_none_or(|r| r.head != head || !r.clean)
1898 {
1899 self.state.status = RunStatus::Blocked;
1900 self.state.repair_grants[index].phase = Phase::Complete;
1901 self.state.event(
1902 "repair-grant",
1903 "head changed since fresh review; repair held",
1904 );
1905 self.state.save()?;
1906 return Ok(());
1907 }
1908 if !verify_current_head(&mut self.state, &winner, &wait).await? {
1909 return Ok(());
1910 }
1911 let outcomes = self.run_gate(&winner).await?;
1912 if verify_inconclusive(&outcomes) {
1913 return Ok(());
1914 }
1915 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1916 if after != head {
1917 self.state.repair_grants[index].review_start = self.state.reviews.len();
1918 self.state.repair_grants[index].head = after;
1919 self.state.repair_grants[index].phase = Phase::Reviewing;
1920 self.state.status = RunStatus::Blocked;
1921 self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1922 self.state.save()?;
1923 return Ok(());
1924 }
1925 let passed = outcomes.iter().all(CommandOutcome::ok);
1926 if passed
1929 && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1930 {
1931 return Ok(());
1932 }
1933 self.state.repair_grants[index].succeeded = passed;
1934 self.state.gate = outcomes;
1935 self.state.gate_ran = true;
1936 self.state.status = if passed {
1937 RunStatus::Gating
1938 } else {
1939 RunStatus::Blocked
1940 };
1941 self.state.repair_grants[index].phase = if !passed
1942 && grant.receipt.request.kind == Kind::Gate
1943 && grant.spent < grant.receipt.request.rounds
1944 {
1945 Phase::Pending
1946 } else {
1947 Phase::Complete
1948 };
1949 if passed && self.state.config.merge.mode == MergeMode::Pr {
1950 self.state.pr_publication_pending = true;
1952 } else if passed {
1953 self.state.status = RunStatus::Ready;
1954 }
1955 self.state.save()?;
1956 if !passed {
1957 continue;
1958 }
1959 if self.state.config.merge.mode != MergeMode::Pr {
1963 self.state.status = RunStatus::Ready;
1964 self.state.event(
1965 "repair-grant",
1966 "repair verified locally; grant carries no merge authorization",
1967 );
1968 self.state.save()?;
1969 return Ok(());
1970 }
1971 return self.merge().await;
1972 }
1973 Phase::Complete => return Ok(()),
1974 }
1975 }
1976 }
1977
1978 async fn execute_graph(&mut self) -> Result<()> {
1979 let archive_home = crate::run::home().canonicalize()?;
1983 if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1984 let repo = self.state.repo.clone();
1985 let _coordination = self.coordinate(&repo).await?;
1986 ensure_setup_config(&mut self.state, &repo).await?;
1987 crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1988 }
1989 self.state.configure_recovery_resume()?;
1990 if self
1991 .state
1992 .repair_grants
1993 .last()
1994 .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1995 {
1996 return self.execute_granted_repair().await;
1997 }
1998 if self
1999 .state
2000 .repair_grants
2001 .last()
2002 .is_some_and(|g| !g.succeeded)
2003 {
2004 self.state.status = RunStatus::Blocked;
2005 self.state.save()?;
2006 return Ok(());
2007 }
2008 self.persist_candidates().await?;
2011 if self.state.base_sync_blocked() {
2014 self.sync_to_base().await?;
2015 if !self.state.base_sync_blocked()
2016 && self.state.status != RunStatus::AlreadyInBase
2017 && !self.resource_halt()?
2018 {
2019 self.finish_after_tally().await?;
2020 }
2021 return Ok(());
2022 }
2023 if self
2024 .state
2025 .merge_decision
2026 .as_ref()
2027 .is_some_and(|decision| !decision.closed)
2028 && self.state.status != RunStatus::Landing
2029 {
2030 if crate::merge_decision::resumable_hold(&self.state) {
2031 self.state.merge = Some(MergeOutcome {
2032 mode: MergeMode::Pr,
2033 ok: false,
2034 detail: self
2035 .state
2036 .merge_decision
2037 .as_ref()
2038 .unwrap()
2039 .target
2040 .pr
2041 .clone(),
2042 empty: false,
2043 });
2044 self.run_land().await?;
2045 self.settle_questions();
2046 return Ok(());
2047 }
2048 bail!(
2049 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2050 );
2051 }
2052 if self.state.config.graph.selected_implementer.is_some()
2053 && self.state.status == RunStatus::Blocked
2054 && self.state.tally.is_none()
2055 && self.state.candidate_preservation.is_empty()
2056 && self.state.interrupted_implementations.is_empty()
2057 {
2058 bail!(
2059 "selected implementation is held; explicitly release the owning task to retry it"
2060 );
2061 }
2062
2063 if self.state.merge_repairs.last().is_some_and(|r| {
2064 matches!(
2065 r.phase,
2066 crate::run::MergeRepairPhase::Requested
2067 | crate::run::MergeRepairPhase::Fixing
2068 | crate::run::MergeRepairPhase::Reviewing
2069 )
2070 }) {
2071 if self.repair_merge_review().await? {
2072 self.run_land().await?;
2073 }
2074 self.settle_questions();
2075 return Ok(());
2076 }
2077 if self.state.pr_publication_pending {
2080 self.merge().await?;
2081 if self.resource_halt()? {
2082 return Ok(());
2083 }
2084 self.state.save()?;
2085 self.settle_questions();
2086 return Ok(());
2087 }
2088 if self.state.status == RunStatus::Stalled {
2101 if self.recover_stall().await? {
2102 self.finish_after_tally().await?;
2103 } else {
2104 self.state.save()?;
2106 }
2107 return Ok(());
2108 }
2109 if self.state.status == RunStatus::Landing
2119 || crate::handover::landing_checkpoint(&self.state)
2120 {
2121 self.state.land_sync_pending = true;
2122 self.state.save()?;
2123 self.run_land().await?;
2124 self.settle_questions();
2129 return Ok(());
2130 }
2131 if self.state.github_text.is_some()
2136 && self.state.merge.is_none()
2137 && self.state.status == RunStatus::Gating
2138 {
2139 self.merge().await?;
2140 if self.resource_halt()? {
2141 return Ok(());
2142 }
2143 self.state.save()?;
2144 return Ok(());
2145 }
2146 if let Err(e) = self.prep().await {
2147 if e.is::<crate::repo_lock::ResourceBlocked>() {
2148 return Err(e);
2149 }
2150 self.state.status = RunStatus::Blocked;
2151 self.state
2152 .prep_failure
2153 .get_or_insert_with(|| format!("{e:#}"));
2154 self.state
2155 .event("prep", format!("preparation stopped: {e:#}"));
2156 self.state.save()?;
2157 return Err(e);
2158 }
2159 if self.park_here()? {
2160 return Ok(());
2161 }
2162 self.advise().await?;
2163 if self.park_here()? {
2164 return Ok(());
2165 }
2166 self.implement().await?;
2167 if self.park_here()? {
2168 return Ok(());
2169 }
2170 if self.state.status == RunStatus::VerifiedNoop {
2174 return Ok(());
2175 }
2176 self.judge().await?;
2177 if self.park_here()? {
2178 return Ok(());
2179 }
2180 self.deliberate().await?;
2181 if self.park_here()? {
2182 return Ok(());
2183 }
2184 self.vote().await?;
2185 if self.park_here()? {
2186 return Ok(());
2187 }
2188 self.tally()?;
2189 if self.state.status == RunStatus::Stalled {
2194 self.state.save()?;
2198 return Ok(());
2199 }
2200 self.finish_after_tally().await?;
2201 Ok(())
2202 }
2203
2204 fn stop_for_resource(&mut self) -> Result<()> {
2209 if self.pause.parked() || self.interrupt.parked() {
2212 self.park_here()?;
2213 return Ok(());
2214 }
2215 if self.state.resource_block.is_some() {
2216 self.state.status = RunStatus::Blocked;
2217 }
2218 self.state.save()
2219 }
2220
2221 fn resource_halt(&mut self) -> Result<bool> {
2223 if self.state.resource_block.is_none() {
2224 return Ok(false);
2225 }
2226 self.stop_for_resource()?;
2227 Ok(true)
2228 }
2229
2230 fn park_here(&mut self) -> Result<bool> {
2237 let parking = self.pause.parked() || self.interrupt.parked();
2238 if !parking && self.state.resource_block.is_some() {
2239 self.stop_for_resource()?;
2240 return Ok(true);
2241 }
2242 if !self.pause.parked() && !self.interrupt.parked() {
2247 return Ok(false);
2248 }
2249 if self.state.parked {
2250 return Ok(true);
2251 }
2252 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2253 if let Some(block) = &mut self.state.resource_block
2254 && matches!(
2255 block.cause,
2256 crate::run::ResourceBlockCause::Cancelled { .. }
2257 )
2258 {
2259 block.cause = crate::run::ResourceBlockCause::Cancelled {
2260 park: cause.clone(),
2261 };
2262 }
2263 let why = match cause.as_ref().map(ParkCause::label) {
2264 Some(reason) => format!(
2265 "parked after `{}` ({reason}) — resume to carry on from here",
2266 self.state.status.as_str()
2267 ),
2268 None => format!(
2269 "parked after `{}` — resume to carry on from here",
2270 self.state.status.as_str()
2271 ),
2272 };
2273 self.state.event("park", why);
2274 self.state
2275 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2276 self.state.parked = true;
2277 self.state.save()?;
2278 Ok(true)
2279 }
2280
2281 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2283 if self.state.prep_failure.is_none() {
2284 return Ok(());
2285 }
2286 let failed_at = self
2287 .state
2288 .events
2289 .iter()
2290 .rev()
2291 .find(|event| event.node == "prep")
2292 .map_or(self.state.updated_at, |event| event.at);
2293 anyhow::ensure!(
2294 approval_at > failed_at,
2295 "preparation retry requires a new explicit release after the failure"
2296 );
2297 self.state.prep_failure = None;
2298 self.state.prep_complete = Some(false);
2299 self.state
2300 .event("prep-retry", "operator approved retry of preparation");
2301 Ok(())
2302 }
2303
2304 pub fn on_pause(&mut self, pause: Pause) {
2306 self.pause = pause;
2307 }
2308
2309 pub fn watch_interrupt(&mut self, pause: Pause) {
2315 self.interrupt = pause;
2316 }
2317
2318 fn settle_questions(&mut self) {
2338 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2339 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2340 }
2341 }
2342
2343 async fn finish_after_tally(&mut self) -> Result<()> {
2346 self.graft().await?;
2349 if self.resource_halt()? {
2350 return Ok(());
2351 }
2352 self.fold_losers().await?;
2353 self.sync_to_base().await?;
2358 if self.hold_base_sync().await? || self.resource_halt()? {
2359 return Ok(());
2360 }
2361 if self.state.status == RunStatus::AlreadyInBase {
2362 return Ok(());
2363 }
2364 self.review_loop().await?;
2365 if self.resource_halt()? {
2366 return Ok(());
2367 }
2368 self.sync_to_base().await?;
2369 if self.hold_base_sync().await? || self.resource_halt()? {
2370 return Ok(());
2371 }
2372 if self.state.status == RunStatus::AlreadyInBase {
2373 return Ok(());
2374 }
2375 self.gate().await?;
2376 self.merge().await?;
2377 if self.resource_halt()? {
2378 return Ok(());
2379 }
2380 self.state.save()?;
2381 Ok(())
2382 }
2383
2384 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2391 let result = match ensure_setup_config(&mut self.state, repo).await {
2392 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2393 Err(e) => Err(e),
2394 };
2395 if let Err(e) = result {
2396 git::worktree_remove_clean(repo, wt).await.ok();
2397 if in_prep {
2398 self.rollback_prep(repo).await;
2399 }
2400 self.state.status = RunStatus::Blocked;
2401 self.state.event(
2402 "setup",
2403 format!("worktree setup failed in {}: {e:#}", wt.display()),
2404 );
2405 self.state.save()?;
2406 return Err(e);
2407 }
2408 Ok(())
2409 }
2410
2411 async fn rollback_prep(&mut self, repo: &Path) {
2413 let root = self.state.worktree_root();
2414 for c in std::mem::take(&mut self.state.candidates) {
2415 if matches!(
2416 git::worktree_remove_clean(repo, &c.worktree).await,
2417 Ok(true)
2418 ) {
2419 git::branch_delete(repo, &c.branch).await.ok();
2420 } else {
2421 self.state.candidates.push(c);
2422 }
2423 }
2424 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2425 for j in 1..=seats {
2426 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2427 .await
2428 .ok();
2429 }
2430 for k in 1..=self.state.config.graph.advisors {
2431 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2432 .await
2433 .ok();
2434 }
2435 }
2436
2437 async fn prep(&mut self) -> Result<()> {
2438 if let Some(reason) = &self.state.prep_failure {
2439 bail!(
2440 "candidate preparation previously failed: {reason}; correct the source before retrying"
2441 );
2442 }
2443 if self
2444 .state
2445 .prep_complete
2446 .unwrap_or(!self.state.candidates.is_empty())
2447 {
2448 return Ok(());
2449 }
2450 let repo = self.state.repo.clone();
2451 let _coordination = self.coordinate(&repo).await?;
2454 let base = self.state.base_commit.clone();
2455 let plan = self.state.sources.plan(&repo, &base).await?;
2456 self.rollback_prep(&repo).await;
2457 if !self.state.candidates.is_empty() {
2458 bail!(
2459 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2460 );
2461 }
2462 self.state.prep_complete = Some(false);
2463 self.state.status = RunStatus::Prep;
2464 self.state.save()?;
2465 let repo = self.state.repo.clone();
2466 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2467 let root = self.state.worktree_root();
2468 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2469
2470 let hooks_dir = self.state.dir().join("hooks");
2473 if self.state.config.blind.commit_msg_hook {
2474 std::fs::create_dir_all(&hooks_dir)
2475 .with_context(|| format!("create {}", hooks_dir.display()))?;
2476 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2477 let path = hooks_dir.join("commit-msg");
2478 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2479 make_executable(&path)?;
2480 git::acquire_worktree_config(&repo).await?;
2488 self.state.enabled_worktree_config = true;
2489 }
2490
2491 ensure_setup_config(&mut self.state, &repo).await?;
2494
2495 for (index, (spec, label)) in self
2496 .roles
2497 .implementers
2498 .clone()
2499 .into_iter()
2500 .zip(labels)
2501 .enumerate()
2502 {
2503 let branch = self.state.branch_for(label);
2504 let worktree = root.join(format!("cand-{label}"));
2505 self.state.event(
2509 "prep-intent",
2510 serde_json::json!({
2511 "branch": branch,
2512 "worktree": worktree,
2513 "start": start,
2514 "picks": plan.picks,
2515 })
2516 .to_string(),
2517 );
2518 self.state.save()?;
2519 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2520 if self.state.config.blind.commit_msg_hook {
2521 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2522 }
2523 git::local_exclude(&worktree, "/.magi/").await?;
2524 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2525 git::worktree_remove_clean(&repo, &worktree).await.ok();
2528 git::branch_delete(&repo, &branch).await.ok();
2529 self.rollback_prep(&repo).await;
2530 self.state.status = RunStatus::Blocked;
2531 self.state
2532 .event("prep", format!("worktree setup failed: {e:#}"));
2533 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2534 self.state.save()?;
2535 return Err(e);
2536 }
2537 self.state.candidates.push(Candidate {
2538 index,
2539 label,
2540 agent: spec.id.clone(),
2541 branch,
2542 worktree,
2543 summary: String::new(),
2544 stat: String::new(),
2545 files: 0,
2546 commits: 0,
2547 empty: false,
2548 failed: None,
2549 verified_noop: None,
2550 duration_ms: 0,
2551 folded: false,
2552 });
2553 self.state.save()?;
2554 for pick in &plan.picks {
2555 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2556 {
2557 self.state.status = RunStatus::Blocked;
2558 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2559 self.state
2560 .event("prep", format!("cannot apply referenced commit: {e}"));
2561 self.state.save()?;
2562 return Err(e);
2563 }
2564 }
2565 }
2566
2567 for j in 1..=self.roles.judges.len() {
2568 let wt = root.join(format!("judge-{j}"));
2569 if !wt.exists() {
2570 git::worktree_add_detached(&repo, &wt, &base).await?;
2571 self.setup_seat_worktree(&repo, &wt, true).await?;
2572 }
2573 }
2574
2575 if self.state.config.graph.advise {
2583 for k in 1..=self.state.config.graph.advisors {
2584 let wt = root.join(format!("advisor-{k}"));
2585 if !wt.exists() {
2586 git::worktree_add_detached(&repo, &wt, &base).await?;
2587 self.setup_seat_worktree(&repo, &wt, true).await?;
2588 }
2589 }
2590 }
2591
2592 let authors: Vec<&str> = self
2597 .roles
2598 .implementers
2599 .iter()
2600 .map(|a| a.id.as_str())
2601 .collect();
2602 let overlap: Vec<String> = self
2603 .roles
2604 .judges
2605 .iter()
2606 .enumerate()
2607 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2608 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2609 .collect();
2610 if !overlap.is_empty() {
2611 let note = format!(
2612 "{} also authored a candidate; blind, but the panel is less \
2613 independent than {} distinct agents would be",
2614 overlap.join(", "),
2615 self.roles.judges.len()
2616 );
2617 self.state.event("prep", note);
2618 }
2619
2620 self.state.event(
2621 "prep",
2622 format!(
2623 "{} candidates, {} judges, base {} ({})",
2624 self.state.candidates.len(),
2625 self.roles.judges.len(),
2626 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2627 self.state.base_branch
2628 ),
2629 );
2630 self.state.prep_complete = Some(true);
2631 self.state.status = RunStatus::Implementing;
2632 self.state.save()?;
2633 Ok(())
2634 }
2635
2636 async fn advise(&mut self) -> Result<()> {
2669 let implement_untouched = self
2670 .state
2671 .candidates
2672 .iter()
2673 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2674 if !self.state.config.graph.advise || self.state.advise_attempted {
2675 return Ok(());
2676 }
2677 if !implement_untouched {
2678 self.state.event(
2679 "advise",
2680 "skipping the design-deliberation stage: at least one \
2681 candidate already shows implementation progress, so this \
2682 run is past the point the stage exists to run before"
2683 .to_owned(),
2684 );
2685 self.state.advise_attempted = true;
2686 self.state.save()?;
2687 return Ok(());
2688 }
2689 let run_id = self.state.id.clone();
2690 let prompts = self.state.config.prompts.clone();
2691 let instruction = self.state.instruction.clone();
2692 let language = self.state.config.graph.language.clone();
2693 let root = self.state.worktree_root();
2694 let n = self.state.config.graph.advisors;
2695 let where_recorded = self.state.dir().join("run.json");
2696
2697 let seats = match self.state.config.advisors() {
2698 Ok(seats) if !seats.is_empty() => seats,
2699 Ok(_) => {
2700 self.state.event(
2701 "advise",
2702 format!(
2703 "[graph] advisors is 0; skipping the design-deliberation \
2704 stage and continuing without a synthesis brief (see {})",
2705 where_recorded.display()
2706 ),
2707 );
2708 self.state.advise_attempted = true;
2709 self.state.save()?;
2710 return Ok(());
2711 }
2712 Err(e) => {
2713 self.state.event(
2714 "advise",
2715 format!(
2716 "could not resolve advisor seats ({e:#}); continuing \
2717 without a design-deliberation brief (see {})",
2718 where_recorded.display()
2719 ),
2720 );
2721 self.state.advise_attempted = true;
2722 self.state.save()?;
2723 return Ok(());
2724 }
2725 };
2726
2727 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2728 let artifacts = agent::artifacts_dir(&self.state.dir());
2729 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2730
2731 let mut jobs = Vec::new();
2732 for (i, spec) in seats.iter().cloned().enumerate() {
2733 let seat_key = format!("advisor-{}", i + 1);
2734 let seat = self.seat(&seat_key, &spec.id);
2735 jobs.push(SeatJob {
2736 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2737 spec,
2738 seat,
2739 cwd: worktrees[i % worktrees.len()].clone(),
2740 timeout,
2741 allow_write: false,
2742 sessions: false,
2743 artifacts: artifacts.clone(),
2744 stem: seat_key,
2745 handover: None,
2746 });
2747 }
2748
2749 self.state.event(
2750 "advise",
2751 format!(
2752 "{} advisor seat(s) sketching a design in parallel",
2753 jobs.len()
2754 ),
2755 );
2756 let mut quota_losses = Vec::new();
2757 let cache = self.state.config.cache_dir();
2758 let ctx = WaveCtx {
2759 carry_seats: false,
2760 pauses: Some((&self.pause, &self.interrupt)),
2761 run: &run_id,
2762 node: "advise",
2763 prompts: &prompts,
2764 cache: cache.as_deref(),
2765 round: None,
2766 };
2767 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2768 let results = ask_json_wave::<Proposal>(
2769 jobs,
2770 Arc::clone(&self.sem),
2771 self.state.config.graph.retries,
2772 &advisor_roster,
2773 &ctx,
2774 &mut quota_losses,
2775 &mut self.state,
2776 &|p: &Proposal| p.validate(),
2777 )
2778 .await;
2779 if self.state.resource_block.is_some() {
2780 return self.stop_for_resource();
2781 }
2782 self.state.quota.extend(quota_losses);
2783
2784 let mut records = Vec::with_capacity(results.len());
2785 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2786 let agent_id = seat.agent.clone();
2787 self.state.seats.insert(seat.key.clone(), seat);
2788 match res {
2789 Ok((proposal, out)) => {
2790 self.state
2791 .event("advise", format!("advisor-{} proposed a design", i + 1));
2792 records.push(advise::AdvisorRecord::proposed(
2793 i + 1,
2794 agent_id,
2795 proposal,
2796 out.duration_ms,
2797 ));
2798 }
2799 Err(e) => {
2800 self.state.event(
2801 "advise",
2802 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2803 );
2804 records.push(advise::AdvisorRecord::failed(
2805 i + 1,
2806 agent_id,
2807 e.to_string(),
2808 ));
2809 }
2810 }
2811 }
2812
2813 let mut advice = advise::Advice {
2814 records,
2815 synthesis: None,
2816 };
2817 if advice.proposals().is_empty() {
2818 self.state.event(
2819 "advise",
2820 "no advisor produced a usable proposal; continuing without a \
2821 synthesis brief"
2822 .to_owned(),
2823 );
2824 } else {
2825 match self
2826 .synthesize_brief(
2827 &advice,
2828 &instruction,
2829 &language,
2830 &worktrees[0],
2831 &artifacts,
2832 &run_id,
2833 &prompts,
2834 cache.as_deref(),
2835 )
2836 .await
2837 {
2838 Ok(Some(text)) => {
2839 self.state.event(
2840 "advise",
2841 "synthesized a design brief for the implementer".to_owned(),
2842 );
2843 advice.synthesis = Some(text);
2844 }
2845 Ok(None) => {
2846 self.state.event(
2847 "advise",
2848 "the synthesis seat produced nothing usable; continuing \
2849 without a design brief"
2850 .to_owned(),
2851 );
2852 }
2853 Err(e) => {
2854 if e.is::<crate::repo_lock::ResourceBlocked>() {
2855 return Err(e);
2856 }
2857 self.state.event(
2858 "advise",
2859 format!("could not synthesize a design brief: {e:#}"),
2860 );
2861 }
2862 }
2863 }
2864 advise::apply_reflection(&mut advice);
2865
2866 self.state.advice = Some(advice);
2867 self.state.advise_attempted = true;
2868 self.state.save()?;
2869 Ok(())
2870 }
2871
2872 #[allow(clippy::too_many_arguments)]
2884 async fn synthesize_brief(
2885 &mut self,
2886 advice: &advise::Advice,
2887 instruction: &str,
2888 language: &str,
2889 cwd: &Path,
2890 artifacts: &Path,
2891 run_id: &str,
2892 prompts: &Prompts,
2893 cache: Option<&Path>,
2894 ) -> Result<Option<String>> {
2895 let chain = agent::pick_chain(
2896 &self.state.config.agents,
2897 self.state.config.roles.synthesizer.as_ref(),
2898 &agent::installed,
2899 "synthesizer",
2900 )?;
2901 let proposals = advice.proposals();
2902 let mut prompt = prompt::with_overlay(
2903 prompt::synthesize_brief(instruction, &proposals, language),
2904 prompts.overlay("advise"),
2905 );
2906 if cache.is_some() {
2907 prompt.push('\n');
2912 prompt.push_str(&prompt::build_cache_note("advise", false));
2913 }
2914 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2915 let mut last = None;
2920 for (n, spec) in chain.iter().enumerate() {
2921 if n > 0 {
2922 self.state
2923 .event("advise", format!("synthesis falling back to {}", spec.id));
2924 }
2925 let mut seat = self.seat("advise-synthesis", &spec.id);
2926 let wait = self.repository_wait();
2927 let outcome = wait
2928 .scope(agent::invoke(
2929 spec,
2930 &mut seat,
2931 &Invocation {
2932 coordination: self.state.config.coordination.as_ref(),
2933 fetched_base: None,
2934 cwd,
2935 prompt: &prompt,
2936 timeout,
2937 allow_write: false,
2938 sessions: false,
2939 artifacts,
2940 stem: &if n == 0 {
2941 "advise-synthesis".to_owned()
2942 } else {
2943 format!("advise-synthesis-{}", spec.id)
2944 },
2945 run: run_id,
2946 node: "advise",
2947 cache_dir: None,
2948 attachments: &[],
2949 writable: &[],
2950 },
2951 ))
2952 .await;
2953 if outcome.is_ok() {
2954 self.state.seats.insert(seat.key.clone(), seat);
2955 }
2956 let advance = agent::chain_advances(&outcome);
2957 last = Some(outcome);
2958 if !advance {
2959 break;
2960 }
2961 }
2962 let out = last.expect("a chain holds at least one agent")?;
2965 if !out.usable() {
2966 return Ok(None);
2967 }
2968 let text =
2969 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2970 Ok((!text.trim().is_empty()).then_some(text))
2971 }
2972
2973 async fn implement(&mut self) -> Result<()> {
2976 self.persist_candidates().await?;
2977 let run_id = self.state.id.clone();
2982 let prompts = self.state.config.prompts.clone();
2983 let todo: Vec<usize> = self
2984 .state
2985 .candidates
2986 .iter()
2987 .enumerate()
2988 .filter(|(_, c)| {
2989 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2990 && c.failed.is_none()
2991 && !c.empty
2992 })
2993 .map(|(i, _)| i)
2994 .collect();
2995 if todo.is_empty() {
2996 return self.after_implement();
2997 }
2998 self.state.status = RunStatus::Implementing;
2999
3000 let language = self.state.config.graph.language.clone();
3001 let github_language = self.state.config.graph.github_language.clone();
3002 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3003 let sessions = self.state.config.graph.sessions;
3004 let artifacts = agent::artifacts_dir(&self.state.dir());
3005 let brief = self
3009 .state
3010 .advice
3011 .as_ref()
3012 .and_then(|a| a.synthesis.as_deref())
3013 .map(str::to_owned);
3014 let attachments = self.state.attachments.clone();
3015
3016 let mut jobs = Vec::new();
3017 for &i in &todo {
3018 let (index, label, worktree) = {
3019 let c = &self.state.candidates[i];
3020 (c.index, c.label, c.worktree.clone())
3021 };
3022 let spec = self.roles.implementers[index].clone();
3023 let seat_key = format!("impl-{label}");
3024 let seat = self.seat(&seat_key, &spec.id);
3025 let instruction = seeded_instruction(&self.state);
3026 jobs.push(SeatJob {
3027 spec,
3028 seat,
3029 prompt: prompt::implement_in(
3030 &instruction,
3031 &worktree.to_string_lossy(),
3032 &language,
3033 &github_language,
3034 brief.as_deref(),
3035 &attachments,
3036 ),
3037 cwd: worktree,
3038 timeout,
3039 allow_write: true,
3040 sessions,
3041 artifacts: artifacts.clone(),
3042 stem: format!("impl-{label}"),
3043 handover: None,
3044 });
3045 }
3046
3047 self.state.event(
3048 "implement",
3049 format!("{} candidates in parallel", jobs.len()),
3050 );
3051 let mut sent = jobs.clone();
3057 let cache = self.state.config.cache_dir();
3058 let ctx = WaveCtx {
3059 carry_seats: false,
3060 pauses: Some((&self.pause, &self.interrupt)),
3061 run: &run_id,
3062 node: "implement",
3063 prompts: &prompts,
3064 cache: cache.as_deref(),
3065 round: None,
3066 };
3067 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3068 if !wave_blocked(&results) {
3069 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3070 .await;
3071 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3072 .await?;
3073 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3074 .await;
3075 }
3076 if !results.is_empty()
3079 && results
3080 .iter()
3081 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3082 {
3083 let losses: Vec<_> = results
3084 .iter()
3085 .map(|(_, seat, out)| {
3086 let AgentOutcome::Quota(o) = out else {
3087 unreachable!()
3088 };
3089 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3090 })
3091 .collect();
3092 self.state.quota.extend(losses.clone());
3093 for (_, seat, _) in results {
3094 self.state.seats.insert(seat.key.clone(), seat);
3095 }
3096 for &i in &todo {
3097 let label = self.state.candidates[i].label;
3098 self.state.interrupted_implementations.insert(label);
3099 self.state
3100 .candidate_preservation
3101 .insert(label, "machine outage awaiting persistence".into());
3102 }
3103 block_machine_outage(&mut self.state, "implement", &losses);
3106 self.state.save()?;
3107 self.persist_candidates().await?;
3108 for &i in &todo {
3109 self.state
3110 .interrupted_implementations
3111 .insert(self.state.candidates[i].label);
3112 }
3113 return self.stop_for_resource();
3114 }
3115 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3116
3117 let selected_failed = self.state.config.graph.selected_implementer.is_some()
3118 && results
3119 .iter()
3120 .any(|(_, _, out)| FailClass::of(out).is_some());
3121 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3122 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3123 if self
3127 .state
3128 .candidate_preservation
3129 .contains_key(&self.state.candidates[i].label)
3130 {
3131 self.state.seats.insert(seat.key.clone(), seat);
3132 }
3133 continue;
3134 }
3135 let seat_key = seat.key.clone();
3136 let agent = seat.agent.clone();
3145 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3146 self.state.seats.insert(seat.key.clone(), seat);
3147 let label = self.state.candidates[i].label;
3148 self.state.interrupted_implementations.remove(&label);
3149
3150 let (summary, duration, failed, verified_claim) = match out {
3151 AgentOutcome::Ok(o) => {
3152 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3153 let failed = (!o.usable()).then(|| {
3154 if o.timed_out {
3155 "agent timed out".to_owned()
3156 } else {
3157 format!(
3158 "agent exited with {}",
3159 crate::notices::exit_status(o.exit_code)
3160 )
3161 }
3162 });
3163 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3164 (text, o.duration_ms, failed, verified_claim)
3165 }
3166 AgentOutcome::Dropped(o) => {
3172 let why = o
3173 .dropped
3174 .as_ref()
3175 .map(|d| d.why.as_str())
3176 .unwrap_or("the CLI ended the stream without delivering its answer");
3177 (
3178 String::new(),
3179 o.duration_ms,
3180 Some(format!("the CLI dropped the stream ({why})")),
3181 None,
3182 )
3183 }
3184 AgentOutcome::Quota(o) => {
3185 self.state.quota.push(QuotaLoss::from_report(
3186 seat_key,
3187 "implement",
3188 o.quota.as_ref(),
3189 ));
3190 (
3191 String::new(),
3192 o.duration_ms,
3193 Some("rate limited (quota); produced no change".to_owned()),
3194 None,
3195 )
3196 }
3197 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3198 (String::new(), 0, Some(e), None)
3199 }
3200 };
3201
3202 let c = &mut self.state.candidates[i];
3205 if !exhausted_the_fallback_chain {
3206 c.agent = agent;
3207 }
3208 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3209 c.duration_ms = duration;
3210 c.failed = failed;
3211 c.verified_noop = verified_claim;
3212 self.state
3213 .candidate_preservation
3214 .insert(label, "awaiting persistence".to_owned());
3215 }
3216 self.state.save()?;
3217 self.persist_candidates().await?;
3218
3219 if resource_blocked {
3220 return self.stop_for_resource();
3221 }
3222 if selected_failed {
3223 self.state.status = RunStatus::Blocked;
3224 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3225 self.state.save()?;
3226 bail!(
3227 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3228 );
3229 }
3230 self.after_implement()
3231 }
3232
3233 async fn persist_candidates(&mut self) -> Result<()> {
3235 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3236 for label in labels {
3237 let i = self
3238 .state
3239 .candidates
3240 .iter()
3241 .position(|c| c.label == label)
3242 .context("pending candidate is missing")?;
3243 let c = self.state.candidates[i].clone();
3244 let interrupted = self.state.interrupted_implementations.contains(&label);
3245 let result = async {
3246 let wait = crate::repo_lock::WaitPolicy::new(
3247 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3248 || false,
3249 );
3250 let _coordination =
3251 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3252 if let Err(e) = git::snapshot_uncommitted(
3255 &c.worktree,
3256 &self
3257 .state
3258 .dir()
3259 .join("artifacts")
3260 .join(format!("candidate-{label}-recovery")),
3261 )
3262 .await
3263 {
3264 self.state.event(
3265 "preserve",
3266 format!("candidate {label} snapshot incomplete: {e:#}"),
3267 );
3268 }
3269 let rescued = git::rescue_commit(
3270 &c.worktree,
3271 &format!("magi: candidate {label} (uncommitted work)"),
3272 )
3273 .await?;
3274 self.state.note_withheld("implement", &rescued.withheld);
3275 let base = &self.state.base_commit;
3276 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3277 let patch = git::diff(&c.worktree, base, "HEAD").await?;
3278 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3279 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3280 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3281 if c.empty && commits > 0 && !patch.trim().is_empty()
3282 && (self.state.judge_skipped || !self.state.judgements.is_empty()
3283 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3284 || self.state.tally.is_some())
3285 {
3286 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3290 }
3291 let c = &mut self.state.candidates[i];
3292 c.commits = commits;
3293 c.stat = stat;
3294 c.files = files;
3295 let no_changes = commits == 0 || patch.trim().is_empty();
3296 c.empty = no_changes && !interrupted;
3297 if interrupted && no_changes {
3298 c.failed = None;
3299 c.verified_noop = None;
3300 }
3301 if !c.empty {
3302 c.failed = None;
3303 c.verified_noop = None;
3304 }
3305 self.state.event(
3306 "preserve",
3307 format!("candidate {label}: {files} files, {commits} commits"),
3308 );
3309 Ok::<_, anyhow::Error>(())
3310 }
3311 .await;
3312 match result {
3313 Ok(()) => {
3314 self.state.candidate_preservation.remove(&label);
3315 if !interrupted || self.state.candidates[i].files > 0 {
3316 self.state.interrupted_implementations.remove(&label);
3317 }
3318 }
3319 Err(e) => {
3320 let reason = format!("candidate {label} preservation failed: {e:#}");
3321 self.state.event("preserve", &reason);
3322 self.state.candidate_preservation.insert(label, reason);
3323 }
3324 }
3325 if self.state.candidate_preservation.is_empty()
3326 && self.state.status == RunStatus::Blocked
3327 && self.state.candidates.iter().all(|c| c.failed.is_none())
3328 {
3329 self.state.status = RunStatus::Implementing;
3333 self.state.event(
3334 "preserve",
3335 "candidate persistence recovered; implementation can continue",
3336 );
3337 }
3338 self.state.save()?;
3339 }
3340 if !self.state.candidate_preservation.is_empty() {
3341 self.state.status = RunStatus::Blocked;
3342 self.state.save()?;
3343 bail!(
3344 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3345 );
3346 }
3347 Ok(())
3348 }
3349
3350 async fn resume_undelivered(
3378 &mut self,
3379 results: &mut [(usize, SeatState, AgentOutcome)],
3380 sent: &[SeatJob],
3381 prompts: &Prompts,
3382 run_id: &str,
3383 ) {
3384 for (wi, seat, out) in results.iter_mut() {
3385 let Some(dropped) = (match &*out {
3386 AgentOutcome::Dropped(o) => o.dropped.clone(),
3387 _ => None,
3388 }) else {
3389 continue;
3390 };
3391 let Some(job) = sent.get(*wi) else { continue };
3392 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3394 self.state.event(
3395 "implement",
3396 format!(
3397 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3398 work is in the tree",
3399 seat.key, dropped.output_tokens, dropped.why
3400 ),
3401 );
3402 continue;
3403 }
3404 if !has_context(&job.spec, seat, job.sessions) {
3412 self.state.event(
3413 "implement",
3414 format!(
3415 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3416 is no session left to resume",
3417 seat.key, dropped.output_tokens, dropped.why
3418 ),
3419 );
3420 continue;
3421 }
3422 self.state.event(
3423 "implement",
3424 format!(
3425 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3426 conversation",
3427 seat.key, dropped.output_tokens, dropped.why
3428 ),
3429 );
3430 let mut retry = job.clone();
3431 retry.seat = seat.clone();
3432 retry.prompt = prompt::resume_after_drop(&dropped.why);
3433 retry.timeout = retry_budget(job.timeout, true);
3434 retry.stem = format!("{}-resume", job.stem);
3435 let cache = self.state.config.cache_dir();
3436 let ctx = WaveCtx {
3437 carry_seats: false,
3438 pauses: Some((&self.pause, &self.interrupt)),
3439 run: run_id,
3440 node: "implement",
3441 prompts,
3442 cache: cache.as_deref(),
3443 round: None,
3444 };
3445 let (resumed_seat, resumed) =
3446 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3447 *seat = resumed_seat;
3448 *out = resumed;
3449 }
3450 }
3451
3452 async fn resume_seat_handovers(
3517 &mut self,
3518 results: &mut [(usize, SeatState, AgentOutcome)],
3519 sent: &mut [SeatJob],
3520 prompts: &Prompts,
3521 run_id: &str,
3522 ) -> Result<()> {
3523 let instruction = seeded_instruction(&self.state);
3524 let language = self.state.config.graph.language.clone();
3525 let github_language = self.state.config.graph.github_language.clone();
3526 let brief = self
3527 .state
3528 .advice
3529 .as_ref()
3530 .and_then(|a| a.synthesis.as_deref())
3531 .map(str::to_owned);
3532 let attachments = self.state.attachments.clone();
3533 for (wi, seat, out) in results.iter_mut() {
3534 let others: BTreeSet<String> = sent
3537 .iter()
3538 .enumerate()
3539 .filter(|(j, _)| j != wi)
3540 .map(|(_, j)| j.spec.id.clone())
3541 .collect();
3542 let Some(job) = sent.get_mut(*wi) else {
3543 continue;
3544 };
3545 let start = self
3550 .roles
3551 .implementer_roster
3552 .iter()
3553 .position(|s| s.id == job.spec.id)
3554 .unwrap_or(0);
3555 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3556 let mut fallback_attempt = 0usize;
3557 let mut prev: Option<FailClass> = None;
3558 while let Some(cur) = FailClass::of(&*out) {
3559 if !should_hand_over(prev.as_ref(), &cur) {
3560 break;
3561 }
3562 let Some(next) =
3563 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3564 .cloned()
3565 else {
3566 break;
3567 };
3568 tried.insert(next.id.clone());
3569 fallback_attempt += 1;
3570
3571 let coordination = self.coordinate(&job.cwd).await?;
3572 if let Ok(r) = git::rescue_commit(
3573 &job.cwd,
3574 &format!(
3575 "magi: candidate {} (uncommitted work before {} fallback)",
3576 seat.key,
3577 if cur == FailClass::Quota {
3578 "quota"
3579 } else {
3580 "handover"
3581 }
3582 ),
3583 )
3584 .await
3585 {
3586 self.state.note_withheld("implement", &r.withheld);
3587 }
3588 drop(coordination);
3589
3590 record_handover(
3591 &mut self.state,
3592 "implement",
3593 &seat.key,
3594 &seat.agent,
3595 &next.id,
3596 &cur,
3597 &fail_reason(&*out),
3598 );
3599 prev = Some(cur.clone());
3600
3601 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3602 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3603 job.spec = next.clone();
3611 let mut retry = job.clone();
3612 retry.seat = new_seat;
3613 retry.prompt = prompt::implement_in(
3614 &instruction,
3615 &job.cwd.to_string_lossy(),
3616 &language,
3617 &github_language,
3618 brief.as_deref(),
3619 &attachments,
3620 );
3621 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3622 let cache = self.state.config.cache_dir();
3623 let ctx = WaveCtx {
3624 carry_seats: false,
3625 pauses: Some((&self.pause, &self.interrupt)),
3626 run: run_id,
3627 node: "implement",
3628 prompts,
3629 cache: cache.as_deref(),
3630 round: None,
3631 };
3632 let (fallback_seat, fallback_out) = run_one(
3633 retry,
3634 Arc::clone(&self.sem),
3635 &ctx,
3636 &mut self.state,
3637 fallback_attempt,
3638 )
3639 .await;
3640 *seat = fallback_seat;
3641 *out = fallback_out;
3642 }
3643 }
3644 Ok(())
3645 }
3646
3647 async fn resume_unconfirmed_commands(
3671 &mut self,
3672 results: &mut [(usize, SeatState, AgentOutcome)],
3673 sent: &[SeatJob],
3674 prompts: &Prompts,
3675 run_id: &str,
3676 ) {
3677 for (wi, seat, out) in results.iter_mut() {
3678 let AgentOutcome::Ok(o) = &*out else {
3679 continue;
3680 };
3681 if !has_unconfirmed_command(&o.commands) {
3682 continue;
3683 }
3684 let Some(job) = sent.get(*wi) else { continue };
3685 if !has_context(&job.spec, seat, job.sessions) {
3686 self.state.event(
3687 "implement",
3688 format!(
3689 "{}: the reply named a command whose own CLI never confirmed the exit \
3690 status of, but there is no session left to resume",
3691 seat.key
3692 ),
3693 );
3694 continue;
3695 }
3696 self.state.event(
3697 "implement",
3698 format!(
3699 "{}: the reply named a command whose own CLI never confirmed the exit \
3700 status of; resuming the conversation",
3701 seat.key
3702 ),
3703 );
3704 let mut retry = job.clone();
3705 retry.seat = seat.clone();
3706 retry.prompt = prompt::resume_incomplete(
3707 "a command in your last reply had no confirmed exit status",
3708 );
3709 retry.timeout = retry_budget(job.timeout, true);
3710 retry.stem = format!("{}-confirm", job.stem);
3711 let cache = self.state.config.cache_dir();
3712 let ctx = WaveCtx {
3713 carry_seats: false,
3714 pauses: Some((&self.pause, &self.interrupt)),
3715 run: run_id,
3716 node: "implement",
3717 prompts,
3718 cache: cache.as_deref(),
3719 round: None,
3720 };
3721 let (resumed_seat, resumed) =
3722 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3723 *seat = resumed_seat;
3724 *out = resumed;
3725 }
3726 }
3727
3728 async fn continue_fix_report(
3749 &mut self,
3750 mut seat: SeatState,
3751 parse_err: String,
3752 job: &SeatJob,
3753 prompts: &Prompts,
3754 run_id: &str,
3755 round: usize,
3756 ) -> (
3757 SeatState,
3758 Option<FixReport>,
3759 Option<String>,
3760 ContinuationRecord,
3761 ) {
3762 let mut last_err = parse_err;
3763 let mut cumulative_wait_ms = 0u64;
3764 let mut attempts = 0usize;
3765 loop {
3766 if !has_context(&job.spec, &seat, job.sessions) {
3767 self.state.event(
3768 "fix",
3769 format!(
3770 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3771 session left to resume into"
3772 ),
3773 );
3774 let outcome = if attempts == 0 {
3775 ContinuationOutcome::NoSession
3776 } else {
3777 ContinuationOutcome::Exhausted
3778 };
3779 return (
3780 seat,
3781 None,
3782 Some(format!("unparsable fix report: {last_err}")),
3783 ContinuationRecord {
3784 attempts,
3785 cumulative_wait_ms,
3786 outcome,
3787 },
3788 );
3789 }
3790 if attempts >= MAX_FIX_CONTINUATIONS {
3791 self.state.event(
3792 "fix",
3793 format!(
3794 "round {round}: fixer's reply still had no adoption report after \
3795 {attempts} continuation(s) ({last_err}); giving up"
3796 ),
3797 );
3798 return (
3799 seat,
3800 None,
3801 Some(format!(
3802 "unparsable fix report after {attempts} continuation(s): {last_err}"
3803 )),
3804 ContinuationRecord {
3805 attempts,
3806 cumulative_wait_ms,
3807 outcome: ContinuationOutcome::Exhausted,
3808 },
3809 );
3810 }
3811 attempts += 1;
3812 self.state.event(
3813 "fix",
3814 format!(
3815 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3816 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3817 ),
3818 );
3819 let mut retry = job.clone();
3820 retry.seat = seat.clone();
3821 retry.prompt = prompt::resume_incomplete(&last_err);
3822 retry.timeout = retry_budget(job.timeout, true);
3823 retry.stem = format!("{}-continue{attempts}", job.stem);
3824 let cache = self.state.config.cache_dir();
3825 let ctx = WaveCtx {
3826 carry_seats: false,
3827 pauses: Some((&self.pause, &self.interrupt)),
3828 run: run_id,
3829 node: "fix",
3830 prompts,
3831 cache: cache.as_deref(),
3832 round: Some(round),
3833 };
3834 let (resumed_seat, resumed_out) = run_one(
3835 retry,
3836 Arc::clone(&self.sem),
3837 &ctx,
3838 &mut self.state,
3839 attempts,
3840 )
3841 .await;
3842 seat = resumed_seat;
3843 match resumed_out {
3844 AgentOutcome::Ok(o) => {
3845 cumulative_wait_ms += o.duration_ms;
3846 match verdict::extract_json::<FixReport>(&o.text) {
3847 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3848 self.state.event(
3849 "fix",
3850 format!(
3851 "round {round}: fixer's adoption report recovered after \
3852 {attempts} continuation(s)"
3853 ),
3854 );
3855 return (
3856 seat,
3857 Some(report),
3858 None,
3859 ContinuationRecord {
3860 attempts,
3861 cumulative_wait_ms,
3862 outcome: ContinuationOutcome::Resumed,
3863 },
3864 );
3865 }
3866 Ok(_) => {
3874 last_err = "the reply parsed, but it reported a command whose own CLI \
3875 never confirmed an exit status"
3876 .to_owned();
3877 }
3878 Err(e) => last_err = e.to_string(),
3879 }
3880 }
3881 AgentOutcome::Quota(o) => {
3882 cumulative_wait_ms += o.duration_ms;
3883 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3884 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3885 self.state.quota.push(loss);
3886 self.state.event(
3887 "fix",
3888 format!(
3889 "round {round}: continuation rate limited (quota); not retrying now"
3890 ),
3891 );
3892 return (
3893 seat,
3894 None,
3895 Some("rate limited (quota) while recovering the fix report".to_owned()),
3896 ContinuationRecord {
3897 attempts,
3898 cumulative_wait_ms,
3899 outcome: ContinuationOutcome::QuotaLost,
3900 },
3901 );
3902 }
3903 AgentOutcome::Dropped(o) => {
3904 cumulative_wait_ms += o.duration_ms;
3905 let why = o
3906 .dropped
3907 .as_ref()
3908 .map(|d| d.why.as_str())
3909 .unwrap_or("the CLI ended the stream without delivering its answer");
3910 last_err = format!("the CLI dropped the stream ({why})");
3911 }
3912 AgentOutcome::Failed(e) => last_err = e,
3913 AgentOutcome::ResourceBlocked(e) => {
3916 return (
3917 seat,
3918 None,
3919 Some(format!("the continuation was not started: {e}")),
3920 ContinuationRecord {
3921 attempts: attempts.saturating_sub(1),
3922 cumulative_wait_ms,
3923 outcome: ContinuationOutcome::NoSession,
3924 },
3925 );
3926 }
3927 }
3928 }
3929 }
3930
3931 fn after_implement(&mut self) -> Result<()> {
3932 if !self.state.candidate_preservation.is_empty() {
3933 bail!("candidate preservation incomplete; judging is unsafe");
3934 }
3935 if self.state.leaks.is_empty() {
3937 let cfg = self.state.config.blind.clone();
3938 let mut leaks = Vec::new();
3939 for c in &self.state.candidates {
3940 let Some(patch) =
3941 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3942 else {
3943 continue;
3944 };
3945 leaks.extend(blind::scan(
3946 &format!("candidate {} patch", c.label),
3947 &patch,
3948 &cfg.vendor_tokens,
3949 ));
3950 }
3951 if !leaks.is_empty() {
3952 let summary = leaks
3953 .iter()
3954 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3955 .collect::<Vec<_>>()
3956 .join(", ");
3957 match cfg.on_leak {
3958 LeakPolicy::Fail => {
3959 self.state.status = RunStatus::Failed;
3960 self.state
3961 .event("blind", format!("vendor text in a patch: {summary}"));
3962 self.state.leaks = leaks;
3963 self.state.save()?;
3964 self.settle_questions();
3965 bail!(
3966 "blind.on_leak = \"fail\" and vendor text reached a \
3967 judged patch: {summary}"
3968 );
3969 }
3970 LeakPolicy::Redact => self.state.event(
3971 "blind",
3972 format!("redacting vendor text for judging: {summary}"),
3973 ),
3974 LeakPolicy::Warn => self.state.event(
3975 "blind",
3976 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3977 ),
3978 }
3979 self.state.leaks = leaks;
3980 }
3981 }
3982
3983 if self.state.viable().is_empty() {
3984 if self.state.all_candidates_verified_noop() {
3985 self.state.status = RunStatus::VerifiedNoop;
3996 self.state.save()?;
3997 self.settle_questions();
3998 return Ok(());
3999 }
4000 self.state.status = RunStatus::Failed;
4001 self.state.save()?;
4002 self.settle_questions();
4003 bail!("no candidate produced a change; nothing to judge");
4004 }
4005 self.state.status = RunStatus::Judging;
4006 self.state.save()?;
4007 Ok(())
4008 }
4009
4010 async fn judge(&mut self) -> Result<()> {
4013 let run_id = self.state.id.clone();
4018 let prompts = self.state.config.prompts.clone();
4019 if !self.state.judgements.is_empty() || self.state.judge_skipped {
4020 return Ok(());
4021 }
4022 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4023 if viable.len() == 1 {
4024 self.state.judge_skipped = true;
4031 self.state.event(
4032 "judge",
4033 format!(
4034 "only candidate {} produced a change; judging skipped",
4035 viable[0].label
4036 ),
4037 );
4038 self.state.save()?;
4039 return Ok(());
4040 }
4041 self.state.status = RunStatus::Judging;
4042
4043 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4044 let language = self.state.config.graph.language.clone();
4045 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4046 let sessions = self.state.config.graph.sessions;
4047 let artifacts = agent::artifacts_dir(&self.state.dir());
4048 let root = self.state.worktree_root();
4049 let base_short = short(&self.state.base_commit);
4050
4051 let mut jobs = Vec::new();
4052 let mut orders = Vec::new();
4053 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4054 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4055 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4056 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4057 let seat_key = format!("judge-{}", j + 1);
4058 let seat = self.seat(&seat_key, &spec.id);
4059 jobs.push(SeatJob {
4060 prompt: prompt::judge(
4061 &self.state.instruction,
4062 &views,
4063 self.roles.judges.len(),
4064 &base_short,
4065 &language,
4066 ),
4067 spec,
4068 seat,
4069 cwd: root.join(format!("judge-{}", j + 1)),
4070 timeout,
4071 allow_write: false,
4072 sessions,
4073 artifacts: artifacts.clone(),
4074 stem: format!("judge-{}", j + 1),
4075 handover: None,
4076 });
4077 }
4078
4079 self.state.event(
4080 "judge",
4081 format!(
4082 "{} judges ranking {} candidates blind",
4083 jobs.len(),
4084 viable.len()
4085 ),
4086 );
4087 let labels_for_check = labels.clone();
4088 let mut quota_losses = Vec::new();
4089 let cache = self.state.config.cache_dir();
4090 let ctx = WaveCtx {
4091 carry_seats: false,
4092 pauses: Some((&self.pause, &self.interrupt)),
4093 run: &run_id,
4094 node: "judge",
4095 prompts: &prompts,
4096 cache: cache.as_deref(),
4097 round: None,
4098 };
4099 let results = ask_json_wave::<Ranking>(
4100 jobs,
4101 Arc::clone(&self.sem),
4102 self.state.config.graph.retries,
4103 &self.roles.judge_roster,
4104 &ctx,
4105 &mut quota_losses,
4106 &mut self.state,
4107 &move |r: &Ranking| r.validate(&labels_for_check),
4108 )
4109 .await;
4110 if self.state.resource_block.is_some() {
4111 return self.stop_for_resource();
4112 }
4113 self.state.quota.extend(quota_losses);
4114
4115 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4116 let agent_id = seat.agent.clone();
4117 self.state.seats.insert(seat.key.clone(), seat);
4118 let mut record = Judgement {
4119 judge: j + 1,
4120 seat: format!("judge-{}", j + 1),
4121 agent: agent_id,
4122 ranking: Vec::new(),
4123 reasons: BTreeMap::new(),
4124 confidence: None,
4125 strengths: BTreeMap::new(),
4126 order: orders[j].clone(),
4127 failed: None,
4128 duration_ms: 0,
4129 };
4130 match res {
4131 Ok((ranking, out)) => {
4132 record.ranking = ranking.normalized();
4133 record.reasons = ranking.reasons;
4134 record.strengths = ranking.strengths;
4135 record.confidence = ranking.confidence;
4136 record.duration_ms = out.duration_ms;
4137 self.state.event(
4138 "judge",
4139 format!(
4140 "judge {} ranked {}",
4141 j + 1,
4142 record.ranking.iter().collect::<String>()
4143 ),
4144 );
4145 }
4146 Err(e) => {
4147 record.failed = Some(e.to_string());
4148 self.state
4149 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4150 }
4151 }
4152 self.state.judgements.push(record);
4153 self.state.save()?;
4154 }
4155 Ok(())
4156 }
4157
4158 async fn deliberate(&mut self) -> Result<()> {
4161 if self.hold_base_sync().await? {
4162 return Ok(());
4163 }
4164 let run_id = self.state.id.clone();
4169 let prompts = self.state.config.prompts.clone();
4170 let rounds = self.state.config.graph.deliberate_rounds;
4171 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4173 return Ok(());
4174 }
4175 let tops: Vec<char> = self
4176 .state
4177 .judgements
4178 .iter()
4179 .filter_map(|j| j.ranking.first().copied())
4180 .collect();
4181 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4182 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4183 self.state.event(
4184 "deliberate",
4185 format!("judges agreed on {} outright; no deliberation", tops[0]),
4186 );
4187 }
4188 self.state.status = RunStatus::Voting;
4189 self.state.save()?;
4190 return Ok(());
4191 }
4192
4193 self.state.status = RunStatus::Deliberating;
4194 self.state.event(
4195 "deliberate",
4196 format!(
4197 "split: first choices were {} — opening {rounds} round(s)",
4198 tops.iter().collect::<String>()
4199 ),
4200 );
4201
4202 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4203 let language = self.state.config.graph.language.clone();
4204 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4205 let sessions = self.state.config.graph.sessions;
4206 let artifacts = agent::artifacts_dir(&self.state.dir());
4207 let root = self.state.worktree_root();
4208 let base_short = short(&self.state.base_commit);
4209
4210 for round in self.state.deliberation.len() + 1..=rounds {
4214 let mut turns: Vec<DeliberationTurn> = Vec::new();
4215 let mut round_losses = Vec::new();
4216 let mut asked = 0;
4217 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4218 if self.state.judgements[j].failed.is_some() {
4219 continue;
4220 }
4221 let seat_key = format!("judge-{}", j + 1);
4222 let spec = self.occupant(&seat_key, spec);
4223 let mut seat = self.seat(&seat_key, &spec.id);
4224 let transcript = self.transcript(&turns, j);
4225 let build = |context: Option<&str>| {
4226 prompt::deliberate(
4227 &self.state.instruction,
4228 context,
4229 &transcript,
4230 round,
4231 rounds,
4232 &language,
4233 )
4234 };
4235 let block = self.candidate_block(&viable, &base_short);
4236 let full = build(Some(&block));
4237 let text = if has_context(&spec, &seat, sessions) {
4238 build(None)
4239 } else {
4240 full.clone()
4241 };
4242 let job = SeatJob {
4243 spec,
4244 seat: seat.clone(),
4245 prompt: text,
4246 cwd: root.join(format!("judge-{}", j + 1)),
4247 timeout,
4248 allow_write: false,
4249 sessions,
4250 artifacts: artifacts.clone(),
4251 stem: format!("delib-{round}-judge-{}", j + 1),
4252 handover: Some(full),
4253 };
4254 let cache = self.state.config.cache_dir();
4255 let ctx = WaveCtx {
4256 carry_seats: false,
4257 pauses: Some((&self.pause, &self.interrupt)),
4258 run: &run_id,
4259 node: "deliberate",
4260 prompts: &prompts,
4261 cache: cache.as_deref(),
4262 round: None,
4263 };
4264 let mut losses = Vec::new();
4268 asked += 1;
4269 let mut results = ask_wave_with::<String>(
4270 vec![job],
4271 Arc::clone(&self.sem),
4272 0,
4273 &self.roles.judge_roster,
4274 &ctx,
4275 &mut losses,
4276 &mut self.state,
4277 &|text: &str| {
4278 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4279 },
4280 )
4281 .await;
4282 if self.state.resource_block.is_some() {
4283 return self.stop_for_resource();
4284 }
4285 round_losses.extend(losses.iter().cloned());
4286 self.state.quota.extend(losses);
4287 let (updated, res, _) = results.pop().expect("one job in, one result out");
4288 seat = updated;
4289 let agent_id = seat.agent.clone();
4290 self.state.seats.insert(seat.key.clone(), seat);
4291 let body = match res {
4292 Ok((body, _)) => body,
4293 Err(e) => {
4296 self.state
4297 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4298 continue;
4299 }
4300 };
4301 let tentative = verdict::extract_json::<Position>(&body)
4302 .ok()
4303 .and_then(|p| p.tentative)
4304 .and_then(|s| s.trim().chars().next())
4305 .map(|c| c.to_ascii_uppercase());
4306 self.state.event(
4307 "deliberate",
4308 format!(
4309 "round {round}: judge {} now favours {}",
4310 j + 1,
4311 tentative.map_or("—".to_owned(), |c| c.to_string())
4312 ),
4313 );
4314 turns.push(DeliberationTurn {
4315 judge: j + 1,
4316 agent: agent_id,
4317 body: blind::sanitize_prose(&body, &self.state.config.blind),
4318 tentative,
4319 });
4320 }
4321 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4322 block_machine_outage(&mut self.state, "deliberate", &round_losses);
4323 return self.stop_for_resource();
4324 }
4325 self.state
4326 .deliberation
4327 .push(DeliberationRound { round, turns });
4328 self.state.save()?;
4329 }
4330
4331 self.state.status = RunStatus::Voting;
4332 self.state.save()?;
4333 Ok(())
4334 }
4335
4336 async fn vote(&mut self) -> Result<()> {
4339 if self.hold_base_sync().await? {
4340 return Ok(());
4341 }
4342 let run_id = self.state.id.clone();
4347 let prompts = self.state.config.prompts.clone();
4348 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4349 return Ok(());
4350 }
4351 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4352 if viable.len() == 1 {
4353 return Ok(());
4354 }
4355 self.state.status = RunStatus::Voting;
4356
4357 let language = self.state.config.graph.language.clone();
4358 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4359 let sessions = self.state.config.graph.sessions;
4360 let artifacts = agent::artifacts_dir(&self.state.dir());
4361 let root = self.state.worktree_root();
4362 let base_short = short(&self.state.base_commit);
4363 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4364
4365 let mut jobs = Vec::new();
4366 let mut seats_at = Vec::new();
4367 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4368 if self
4369 .state
4370 .judgements
4371 .get(j)
4372 .is_some_and(|r| r.failed.is_some())
4373 {
4374 continue;
4375 }
4376 let seat_key = format!("judge-{}", j + 1);
4377 let spec = self.occupant(&seat_key, spec);
4378 let seat = self.seat(&seat_key, &spec.id);
4379 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4380 let text = if has_context(&spec, &seat, sessions) {
4381 prompt::final_vote(&viable, &language)
4382 } else {
4383 full.clone()
4384 };
4385 jobs.push(SeatJob {
4386 spec,
4387 seat,
4388 prompt: text,
4389 cwd: root.join(format!("judge-{}", j + 1)),
4390 timeout,
4391 allow_write: false,
4392 sessions,
4393 artifacts: artifacts.clone(),
4394 stem: format!("vote-judge-{}", j + 1),
4395 handover: Some(full),
4396 });
4397 seats_at.push(j);
4398 }
4399
4400 self.state.event(
4401 "vote",
4402 format!(
4403 "collecting {} final votes one by one, privately",
4404 jobs.len()
4405 ),
4406 );
4407 let allowed = viable.clone();
4408 let mut quota_losses = Vec::new();
4409 let cache = self.state.config.cache_dir();
4410 let ctx = WaveCtx {
4411 carry_seats: false,
4412 pauses: Some((&self.pause, &self.interrupt)),
4413 run: &run_id,
4414 node: "vote",
4415 prompts: &prompts,
4416 cache: cache.as_deref(),
4417 round: None,
4418 };
4419 let results = ask_json_wave::<FinalVote>(
4420 jobs,
4421 Arc::clone(&self.sem),
4422 self.state.config.graph.retries,
4423 &self.roles.judge_roster,
4424 &ctx,
4425 &mut quota_losses,
4426 &mut self.state,
4427 &move |v: &FinalVote| match v.label() {
4428 Some(c) if allowed.contains(&c) => Ok(()),
4429 other => bail!("vote {other:?} is not one of {allowed:?}"),
4430 },
4431 )
4432 .await;
4433 if self.state.resource_block.is_some() {
4434 return self.stop_for_resource();
4435 }
4436 self.state.quota.extend(quota_losses);
4437
4438 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4439 let agent_id = seat.agent.clone();
4440 self.state.seats.insert(seat.key.clone(), seat);
4441 let initial = self
4442 .state
4443 .judgements
4444 .get(j)
4445 .and_then(|r| r.ranking.first().copied());
4446 let mut record = VoteRecord {
4447 judge: j + 1,
4448 agent: agent_id,
4449 vote: None,
4450 reason: String::new(),
4451 changed: false,
4452 };
4453 match res {
4454 Ok((v, _)) => {
4455 record.vote = v.label();
4456 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4457 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4458 self.state.event(
4459 "vote",
4460 format!(
4461 "judge {} voted {}{}",
4462 j + 1,
4463 record.vote.unwrap_or('?'),
4464 if record.changed { " (changed)" } else { "" }
4465 ),
4466 );
4467 }
4468 Err(e) => {
4469 self.state
4470 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4471 }
4472 }
4473 self.state.votes.push(record);
4474 self.state.save()?;
4475 }
4476 Ok(())
4477 }
4478
4479 fn tally(&mut self) -> Result<()> {
4482 if self.state.tally.is_some() {
4483 return Ok(());
4484 }
4485 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4486 let tops: Vec<char> = self
4487 .state
4488 .judgements
4489 .iter()
4490 .filter_map(|j| j.ranking.first().copied())
4491 .collect();
4492 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4493
4494 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4497 let mut cast: Vec<char> = Vec::new();
4498 for (i, j) in self.state.judgements.iter().enumerate() {
4499 let vote = self
4500 .state
4501 .votes
4502 .iter()
4503 .find(|v| v.judge == i + 1)
4504 .and_then(|v| v.vote)
4505 .or_else(|| j.ranking.first().copied());
4506 if let Some(v) = vote {
4507 *first_choice.entry(v).or_insert(0) += 1;
4508 cast.push(v);
4509 }
4510 }
4511
4512 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4513 for j in &self.state.judgements {
4514 let n = j.ranking.len();
4515 for (pos, label) in j.ranking.iter().enumerate() {
4516 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4517 }
4518 }
4519
4520 let best = first_choice.values().copied().max().unwrap_or(0);
4521 let mut leaders: Vec<char> = first_choice
4522 .iter()
4523 .filter(|(_, v)| **v == best)
4524 .map(|(k, _)| *k)
4525 .collect();
4526 let mut tie_break = None;
4527 if leaders.len() > 1 {
4528 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4529 let borda_leaders: Vec<char> = leaders
4530 .iter()
4531 .copied()
4532 .filter(|l| borda[l] == top_borda)
4533 .collect();
4534 tie_break = Some(if borda_leaders.len() == 1 {
4535 format!(
4536 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4537 leaders.len()
4538 )
4539 } else {
4540 format!(
4541 "{} way tie on both first-choice votes and Borda points, broken by label order",
4542 leaders.len()
4543 )
4544 });
4545 leaders = borda_leaders;
4546 leaders.sort_unstable();
4547 }
4548 let winner = *leaders
4549 .first()
4550 .or(viable.first())
4551 .context("no candidate to declare a winner from")?;
4552
4553 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4554 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4555 let deliberated = !self.state.deliberation.is_empty();
4556
4557 let quota_seats: std::collections::BTreeSet<&str> =
4561 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4562 let mut present = 0usize;
4563 for (i, j) in self.state.judgements.iter().enumerate() {
4564 if quota_seats.contains(j.seat.as_str()) {
4565 continue;
4566 }
4567 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4568 let voted = self
4569 .state
4570 .votes
4571 .iter()
4572 .any(|v| v.judge == i + 1 && v.vote.is_some());
4573 if ranked || voted {
4574 present += 1;
4575 }
4576 }
4577 let needs_quorum = viable.len() > 1;
4583 let judges_total = if needs_quorum {
4584 self.roles.judges.len()
4585 } else {
4586 0
4587 };
4588 let quorum = if needs_quorum {
4589 judges_total / 2 + 1
4590 } else {
4591 0
4592 };
4593 let met_quorum = !needs_quorum || present >= quorum;
4594 let uncontested = (!needs_quorum).then(|| {
4595 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4596 });
4597
4598 self.state.event(
4599 "tally",
4600 match &uncontested {
4601 Some(reason) => format!("winner {winner} — {reason}"),
4602 None => format!(
4603 "winner {winner} — votes {} | initial {} | {} changed | \
4604 {present}/{judges_total} judges{}",
4605 first_choice
4606 .iter()
4607 .map(|(k, v)| format!("{k}:{v}"))
4608 .collect::<Vec<_>>()
4609 .join(" "),
4610 if unanimous_initial {
4611 "unanimous"
4612 } else {
4613 "split"
4614 },
4615 changed_votes,
4616 if met_quorum {
4617 String::new()
4618 } else {
4619 format!(" — below quorum ({quorum} required)")
4620 },
4621 ),
4622 },
4623 );
4624 if !met_quorum {
4625 self.state.event(
4626 "stall",
4627 format!(
4628 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4629 the run stops here, resumable"
4630 ),
4631 );
4632 }
4633 self.state.tally = Some(Tally {
4634 first_choice,
4635 borda,
4636 winner,
4637 rankings: tops.len(),
4638 unanimous_initial,
4639 deliberated,
4640 changed_votes,
4641 unanimous_final,
4642 tie_break,
4643 judges: judges_total,
4644 present,
4645 quorum,
4646 met_quorum,
4647 uncontested,
4648 });
4649 self.state.status = if met_quorum {
4650 RunStatus::Reviewing
4651 } else {
4652 RunStatus::Stalled
4653 };
4654 self.state.save()?;
4655 Ok(())
4656 }
4657
4658 #[allow(clippy::too_many_lines)]
4679 async fn recover_stall(&mut self) -> Result<bool> {
4680 let run_id = self.state.id.clone();
4685 let prompts = self.state.config.prompts.clone();
4686 let quota_seats: BTreeSet<&str> =
4691 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4692 let absent: Vec<String> = self
4693 .state
4694 .judgements
4695 .iter()
4696 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4697 .map(|j| j.seat.clone())
4698 .collect();
4699 if absent.is_empty() {
4700 return Ok(false);
4701 }
4702 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4703 if viable.len() <= 1 {
4704 return Ok(false);
4705 }
4706 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4707 let language = self.state.config.graph.language.clone();
4708 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4709 let sessions = self.state.config.graph.sessions;
4710 let artifacts = agent::artifacts_dir(&self.state.dir());
4711 let root = self.state.worktree_root();
4712 let base_short = short(&self.state.base_commit);
4713 let candidates: Vec<Candidate> = viable.clone();
4714
4715 let mut positions: Vec<usize> = absent
4717 .iter()
4718 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4719 .collect();
4720 if positions.is_empty() {
4721 return Ok(false);
4722 }
4723 positions.sort_unstable();
4724 positions.dedup();
4725
4726 let mut judge_jobs = Vec::new();
4728 for &j in &positions {
4729 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4730 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4731 let seat_key = format!("judge-{}", j + 1);
4732 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4733 let seat = self.seat(&seat_key, &spec.id);
4734 judge_jobs.push(SeatJob {
4735 spec,
4736 seat,
4737 prompt: prompt::judge(
4738 &self.state.instruction,
4739 &views,
4740 self.roles.judges.len(),
4741 &base_short,
4742 &language,
4743 ),
4744 cwd: root.join(seat_key),
4745 timeout,
4746 allow_write: false,
4747 sessions,
4748 artifacts: artifacts.clone(),
4749 stem: format!("judge-{}-recover", j + 1),
4750 handover: None,
4751 });
4752 }
4753
4754 let labels_for_check = labels.clone();
4755 let mut judge_losses = Vec::new();
4756 let retries = self.state.config.graph.retries;
4757 let cache = self.state.config.cache_dir();
4758 let ctx = WaveCtx {
4759 carry_seats: false,
4760 pauses: Some((&self.pause, &self.interrupt)),
4761 run: &run_id,
4762 node: "judge",
4763 prompts: &prompts,
4764 cache: cache.as_deref(),
4765 round: None,
4766 };
4767 let results = ask_json_wave::<Ranking>(
4768 judge_jobs,
4769 Arc::clone(&self.sem),
4770 retries,
4771 &self.roles.judge_roster,
4772 &ctx,
4773 &mut judge_losses,
4774 &mut self.state,
4775 &move |r: &Ranking| r.validate(&labels_for_check),
4776 )
4777 .await;
4778 if self.state.resource_block.is_some() {
4779 self.stop_for_resource()?;
4780 return Ok(false);
4781 }
4782
4783 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4785 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4786 let agent_id = seat.agent.clone();
4787 self.state.seats.insert(seat.key.clone(), seat);
4788 let record = &mut self.state.judgements[j];
4789 match res {
4790 Ok((ranking, out)) => {
4791 record.agent = agent_id;
4792 record.ranking = ranking.normalized();
4793 record.reasons = ranking.reasons;
4794 record.strengths = ranking.strengths;
4795 record.confidence = ranking.confidence;
4796 record.failed = None;
4797 record.duration_ms = out.duration_ms;
4798 recovered.insert(j);
4799 self.state.event(
4800 "recover",
4801 format!("judge {} ranked again after the limit", j + 1),
4802 );
4803 }
4804 Err(e) => {
4805 self.state
4806 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4807 }
4808 }
4809 }
4810
4811 let mut vote_jobs = Vec::new();
4813 let mut vote_pos: Vec<usize> = Vec::new();
4814 for &j in &recovered {
4815 let seat_key = format!("judge-{}", j + 1);
4816 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4817 let seat = self.seat(&seat_key, &spec.id);
4818 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4819 let text = if has_context(&spec, &seat, sessions) {
4820 prompt::final_vote(&labels, &language)
4821 } else {
4822 full.clone()
4823 };
4824 vote_jobs.push(SeatJob {
4825 spec,
4826 seat,
4827 prompt: text,
4828 cwd: root.join(seat_key),
4829 timeout,
4830 allow_write: false,
4831 sessions,
4832 artifacts: artifacts.clone(),
4833 stem: format!("vote-judge-{}-recover", j + 1),
4834 handover: Some(full),
4835 });
4836 vote_pos.push(j);
4837 }
4838 let allowed = labels.clone();
4839 let mut vote_losses = Vec::new();
4840 let vote_retries = self.state.config.graph.retries;
4841 let vote_cache = self.state.config.cache_dir();
4842 let ctx = WaveCtx {
4843 carry_seats: false,
4844 pauses: Some((&self.pause, &self.interrupt)),
4845 run: &run_id,
4846 node: "vote",
4847 prompts: &prompts,
4848 cache: vote_cache.as_deref(),
4849 round: None,
4850 };
4851 let votes = ask_json_wave::<FinalVote>(
4852 vote_jobs,
4853 Arc::clone(&self.sem),
4854 vote_retries,
4855 &self.roles.judge_roster,
4856 &ctx,
4857 &mut vote_losses,
4858 &mut self.state,
4859 &move |v: &FinalVote| match v.label() {
4860 Some(c) if allowed.contains(&c) => Ok(()),
4861 other => bail!("vote {other:?} is not one of {allowed:?}"),
4862 },
4863 )
4864 .await;
4865 if self.state.resource_block.is_some() {
4866 self.stop_for_resource()?;
4867 return Ok(false);
4868 }
4869 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4870 let agent_id = seat.agent.clone();
4871 self.state.seats.insert(seat.key.clone(), seat);
4872 match res {
4873 Ok((v, _)) => {
4874 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4875 rec.vote = v.label();
4876 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4877 } else {
4878 self.state.votes.push(VoteRecord {
4879 judge: j + 1,
4880 agent: agent_id,
4881 vote: v.label(),
4882 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4883 changed: false,
4884 });
4885 }
4886 self.state.event(
4887 "recover",
4888 format!("judge {} voted again after the limit", j + 1),
4889 );
4890 }
4891 Err(e) => {
4892 self.state
4893 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4894 }
4895 }
4896 }
4897
4898 let recovered_keys: BTreeSet<String> = recovered
4902 .iter()
4903 .map(|&j| format!("judge-{}", j + 1))
4904 .collect();
4905 self.state
4906 .quota
4907 .retain(|q| !recovered_keys.contains(&q.seat));
4908 for loss in judge_losses.into_iter().chain(vote_losses) {
4912 if recovered_keys.contains(&loss.seat) {
4913 continue;
4914 }
4915 self.state.quota.retain(|q| q.seat != loss.seat);
4916 self.state.quota.push(loss);
4917 }
4918
4919 self.state.tally = None;
4921 self.tally()?;
4922 Ok(self
4923 .state
4924 .tally
4925 .as_ref()
4926 .map(|t| t.met_quorum)
4927 .unwrap_or(false))
4928 }
4929
4930 async fn graft(&mut self) -> Result<()> {
4946 let Err(e) = self.graft_inner().await else {
4947 return Ok(());
4948 };
4949 if e.is::<crate::repo_lock::ResourceBlocked>() {
4950 return Err(e);
4951 }
4952 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4953 self.state.status = RunStatus::Blocked;
4954 self.state.event("graft", format!("blocked: {e:#}"));
4955 self.state.save()?;
4956 return Err(e);
4957 }
4958 if let Some(rec) = self.state.graft.clone()
4963 && !rec.outcome.settled()
4964 {
4965 if let Some(winner) = self.state.winner().cloned() {
4966 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4967 if re.is::<crate::repo_lock::ResourceBlocked>() {
4968 return Err(re);
4969 }
4970 self.state.status = RunStatus::Blocked;
4971 self.state.event("graft", format!("blocked: {re:#}"));
4972 self.state.save()?;
4973 return Err(re);
4974 }
4975 }
4976 if let Some(g) = self.state.graft.as_mut() {
4977 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4978 }
4979 }
4980 self.state.event("graft", format!("skipped: {e:#}"));
4981 if let Err(e) = self.state.save() {
4982 tracing::warn!("save after graft: {e:#}");
4983 }
4984 Ok(())
4985 }
4986
4987 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4990 let _coordination = self.coordinate(wt).await?;
4991 crate::recovery::restore_repair(wt, pre)
4992 .await
4993 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4994 }
4995
4996 fn graft_skipped(&mut self, why: &str) {
4998 let message = format!("skipped: {why}");
4999 if !self
5000 .state
5001 .events
5002 .iter()
5003 .any(|e| e.node == "graft" && e.message == message)
5004 {
5005 self.state.event("graft", message);
5006 }
5007 }
5008
5009 async fn graft_inner(&mut self) -> Result<()> {
5010 if let Some(rec) = self.state.graft.clone()
5013 && !rec.outcome.settled()
5014 {
5015 if self.state.reviews.is_empty()
5019 && let Some(winner) = self.state.winner().cloned()
5020 {
5021 self.graft_restore(&winner.worktree, &rec.pre).await?;
5022 }
5023 if rec.outcome == GraftOutcome::WaitingForRestore {
5024 self.state.graft = None;
5025 self.state.save()?;
5026 } else {
5027 if let Some(g) = self.state.graft.as_mut() {
5028 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5029 }
5030 self.state.event(
5031 "graft",
5032 "rolled back: the previous round was interrupted; the winner is untouched",
5033 );
5034 self.state.save()?;
5035 return Ok(());
5036 }
5037 }
5038 let plan = match graft::plan(&self.state) {
5039 Ok(plan) => plan,
5040 Err(skip) => {
5041 if skip != graft::Skip::AlreadyDone {
5042 self.graft_skipped(skip.reason());
5043 self.state.save()?;
5044 }
5045 return Ok(());
5046 }
5047 };
5048 let Some(winner) = self.state.winner().cloned() else {
5049 return Ok(());
5050 };
5051 let wt = winner.worktree.clone();
5052 if !git::is_clean(&wt).await? {
5053 self.graft_skipped("the winner's worktree has uncommitted changes");
5054 self.state.save()?;
5055 return Ok(());
5056 }
5057 let pre = git::rev_parse(&wt, "HEAD").await?;
5058 self.state.graft = Some(GraftRecord {
5061 pre: pre.clone(),
5062 post: None,
5063 lines: None,
5064 limit: None,
5065 agent: None,
5066 outcome: GraftOutcome::Running,
5067 requested: graft::requested(&plan),
5068 adoption: None,
5069 adoption_error: None,
5070 });
5071 self.state.event(
5072 "graft",
5073 format!(
5074 "porting strengths of candidate(s) {} onto {}",
5075 plan.strengths
5076 .iter()
5077 .map(|(l, _, _)| l.to_string())
5078 .collect::<Vec<_>>()
5079 .join(", "),
5080 plan.winner
5081 ),
5082 );
5083 self.state.save()?;
5084
5085 let fix_prompt = prompt::graft(
5086 &self.state.instruction,
5087 plan.winner,
5088 &plan.strengths,
5089 &self.state.config.graph.language,
5090 );
5091 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5092 let sessions = self.state.config.graph.sessions;
5093 let artifacts = agent::artifacts_dir(&self.state.dir());
5094 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5095 let (_, seat, out) = self
5096 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5097 SeatJob {
5098 prompt: fix_prompt.clone(),
5099 spec,
5100 seat,
5101 cwd: wt.clone(),
5102 timeout,
5103 allow_write: true,
5104 sessions,
5105 artifacts: artifacts.clone(),
5106 stem: "graft".to_owned(),
5107 handover: None,
5108 }
5109 })
5110 .await;
5111 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5112 self.state.graft.as_mut().expect("graft record").outcome =
5115 GraftOutcome::WaitingForRestore;
5116 return self.stop_for_resource();
5117 }
5118 let mut why: Option<String> = None;
5119 match &out {
5120 AgentOutcome::Ok(o) if agent::output_advances(o) => {
5121 why = Some("the fixer produced nothing usable".to_owned());
5122 }
5123 AgentOutcome::Ok(_) => {}
5124 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5125 AgentOutcome::Quota(o) => {
5126 self.state.quota.push(QuotaLoss::from_report(
5127 seat.key.clone(),
5128 "graft",
5129 o.quota.as_ref(),
5130 ));
5131 why = Some("rate limited (quota)".to_owned());
5132 }
5133 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5134 why = Some(format!(
5135 "the fixer failed: {}",
5136 e.lines()
5137 .next()
5138 .unwrap_or("")
5139 .chars()
5140 .take(160)
5141 .collect::<String>()
5142 ));
5143 }
5144 }
5145 let agent_id = seat.agent.clone();
5146 self.state.seats.insert(seat.key.clone(), seat);
5147
5148 let mut post = None;
5149 let mut lines = None;
5150 let _coordination = self.coordinate(&wt).await?;
5151 if why.is_none() {
5152 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5153 if let Ok(r) = &rescued {
5154 self.state.note_withheld("graft", &r.withheld);
5155 }
5156 let head = git::rev_parse(&wt, "HEAD").await?;
5157 let dirty = match &rescued {
5161 Ok(r) => git::status_porcelain(&wt)
5162 .await
5163 .map(|st| {
5164 st.lines().any(|l| {
5165 !r.withheld
5166 .iter()
5167 .any(|w| l.trim_end().ends_with(w.path.as_str()))
5168 })
5169 })
5170 .unwrap_or(true),
5171 Err(_) => true,
5172 };
5173 if dirty {
5174 why = Some("the fixer left changes that could not be committed".to_owned());
5175 } else if head == pre {
5176 why = Some("the fixer changed nothing".to_owned());
5177 } else {
5178 lines = git::changed_lines(&wt, &pre, &head).await;
5179 post = Some(head);
5180 }
5181 }
5182 let (adoption, adoption_error) = match &out {
5183 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5184 _ => (None, None),
5185 };
5186 let requested = graft::requested(&plan);
5187 let outcome = match why {
5188 None => {
5189 self.state.event(
5190 "graft",
5191 format!(
5192 "applied: {}",
5193 lines.map_or_else(
5194 || "changed lines unavailable".to_owned(),
5195 |n| format!("{n} changed lines")
5196 )
5197 ),
5198 );
5199 GraftOutcome::Applied
5200 }
5201 Some(why) => {
5202 crate::recovery::restore_repair(&wt, &pre)
5205 .await
5206 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5207 self.state.event(
5208 "graft",
5209 format!("rolled back: {why}; the winner is untouched"),
5210 );
5211 GraftOutcome::RolledBack(why)
5212 }
5213 };
5214 self.state.graft = Some(GraftRecord {
5215 pre,
5216 post,
5217 lines,
5218 limit: None,
5219 agent: Some(agent_id),
5220 outcome,
5221 requested,
5222 adoption,
5223 adoption_error,
5224 });
5225 self.state.save()?;
5226 Ok(())
5227 }
5228
5229 async fn fold_losers(&mut self) -> Result<()> {
5232 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5233 return Ok(());
5234 };
5235 if !self.state.candidate_preservation.is_empty() {
5236 bail!("candidate preservation incomplete; refusing cleanup");
5237 }
5238 let repo = self.state.repo.clone();
5239 let _coordination = self.coordinate(&repo).await?;
5240 let mut folded = Vec::new();
5241 for i in 0..self.state.candidates.len() {
5242 let c = &self.state.candidates[i];
5243 if c.label == winner || c.folded {
5244 continue;
5245 }
5246 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5247 remove_preserved_worktree(&repo, &wt).await?;
5248 if git::branch_exists(&repo, &branch).await?
5249 && !git::branch_delete(&repo, &branch).await?
5250 {
5251 bail!("candidate branch removal refused");
5252 }
5253 self.state.candidates[i].folded = true;
5254 self.state.save()?;
5255 folded.push(label.to_string());
5256 }
5257 let root = self.state.worktree_root();
5259 for j in 1..=self.roles.judges.len() {
5260 let wt = root.join(format!("judge-{j}"));
5261 if wt.exists() {
5262 remove_preserved_worktree(&repo, &wt).await?;
5263 }
5264 }
5265 if self.state.config.graph.advise {
5268 for k in 1..=self.state.config.graph.advisors {
5269 let wt = root.join(format!("advisor-{k}"));
5270 if wt.exists() {
5271 remove_preserved_worktree(&repo, &wt).await?;
5272 }
5273 }
5274 }
5275 if !folded.is_empty() {
5276 self.state
5277 .event("fold", format!("folded candidates {}", folded.join(", ")));
5278 self.state.save()?;
5279 }
5280 Ok(())
5281 }
5282
5283 async fn hold_base_sync(&mut self) -> Result<bool> {
5286 self.hold_base_sync_as(None).await
5287 }
5288
5289 async fn hold_base_sync_kind(
5290 &mut self,
5291 kind: crate::base_sync_recovery::StopKind,
5292 ) -> Result<()> {
5293 self.hold_base_sync_as(Some(kind)).await?;
5294 Ok(())
5295 }
5296
5297 async fn hold_base_sync_as(
5298 &mut self,
5299 kind: Option<crate::base_sync_recovery::StopKind>,
5300 ) -> Result<bool> {
5301 if !self.state.base_sync_blocked() {
5302 self.state.base_sync_recovery.stop = None;
5303 return Ok(false);
5304 }
5305 let head = match self.state.winner() {
5306 Some(w) => git::rev_parse(&w.worktree, "HEAD")
5307 .await
5308 .unwrap_or_default(),
5309 None => String::new(),
5310 };
5311 if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5312 && !authorization.consumed
5313 && authorization.stop.head != head
5314 {
5315 authorization.refuse(format!(
5316 "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5317 authorization.stop.head
5318 ));
5319 }
5320 self.state.record_base_sync_stop(&head);
5321 if let Some(kind) = kind {
5322 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5323 if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5324 stop.kind = kind;
5325 }
5326 }
5327 self.state.save()?;
5328 Ok(true)
5329 }
5330
5331 async fn sync_to_base(&mut self) -> Result<()> {
5373 if self.state.status == RunStatus::AlreadyInBase {
5374 return Ok(());
5375 }
5376 self.hold_base_sync().await?;
5377 let conflicted = self
5378 .state
5379 .base_sync
5380 .as_ref()
5381 .is_some_and(|s| s.conflict.is_some());
5382 let Some(winner) = self.state.winner().cloned() else {
5383 return Ok(());
5384 };
5385
5386 crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5387
5388 let repo = self.state.repo.clone();
5389 let remote = self.state.config.merge.remote.clone();
5390 let base_branch = self.state.base_branch.clone();
5391 let tracking = format!("{remote}/{base_branch}");
5392
5393 let fetched_base =
5394 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5395 let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5400 return Ok(());
5401 };
5402
5403 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5404 self.state.ensure_base_episode();
5405 let mut pending = self
5406 .state
5407 .base_sync_recovery
5408 .operations
5409 .last()
5410 .filter(|o| !o.finished)
5411 .cloned();
5412 let scratch = self.state.dir().join("base-sync");
5413 if git::rebase_in_progress(&scratch).await {
5414 if pending.is_none()
5417 && self.state.base_sync_recovery.operations.is_empty()
5418 && !conflicted
5419 && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5420 .await
5421 .ok()
5422 .as_deref()
5423 == Some(head.as_str())
5424 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5425 && git::is_clean(&winner.worktree).await?
5426 {
5427 let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5428 if self.state.base_sync.is_none() {
5429 self.state.base_sync = Some(BaseSync {
5430 tip: saved_tip.clone(),
5431 behind: 0,
5432 attempts: 0,
5433 conflict: None,
5434 already_in: None,
5435 });
5436 }
5437 self.state.reserve_base_operation(&head, &saved_tip);
5438 self.state.event(
5439 "base-recovery",
5440 "imported standing legacy rebase as a spent synchronization reservation",
5441 );
5442 self.state.save()?;
5443 pending = self.state.base_sync_recovery.operations.last().cloned();
5444 }
5445 let expected = pending.as_ref().map(|op| op.tip.as_str());
5446 if expected.is_none()
5447 || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5448 {
5449 let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5450 if self.state.base_sync.is_none() {
5451 self.state.base_sync = Some(BaseSync {
5452 tip: tip.clone(),
5453 behind: 0,
5454 attempts: 0,
5455 conflict: None,
5456 already_in: None,
5457 });
5458 }
5459 self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5460 self.state.event("land", why);
5461 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5462 .await?;
5463 return Ok(());
5464 }
5465 }
5466 if let Some(op) = &pending {
5467 let mut replay_proven =
5469 head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5470 if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5471 let wait = repository_wait(&self.state, None);
5472 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5473 match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5474 Ok(true) => {
5475 self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5476 self.state.save()?;
5477 git::worktree_remove(&repo, &scratch).await.ok();
5478 replay_proven = true;
5479 }
5480 Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5481 Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5482 }
5483 }
5484 if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5485 if self.state.foreign_checkout.is_some() {
5486 replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5487 .await
5488 .ok()
5489 .as_deref()
5490 == Some(op.head.as_str())
5491 && git::rev_parse(&repo, &format!("{head}^2"))
5492 .await
5493 .ok()
5494 .as_deref()
5495 == Some(op.tip.as_str());
5496 } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5497 .await
5498 .is_ok_and(|m| m.is_empty())
5499 && let Ok(added) =
5500 crate::reconcile::origin_missing(&repo, &op.head, &head).await
5501 {
5502 replay_proven = true;
5503 for commit in added {
5504 if !git::is_ancestor(&repo, &commit, &op.tip).await {
5505 replay_proven = false;
5506 break;
5507 }
5508 }
5509 replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5510 <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5511 }
5512 }
5513 if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5517 || (head != op.head
5518 && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5519 .await?
5520 .is_empty())
5521 || (!git::is_clean(&winner.worktree).await?
5522 && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5523 .await
5524 .is_ok_and(|o| o.ok()))
5525 || !replay_proven
5526 {
5527 let why = format!(
5528 "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5529 op.id
5530 );
5531 let sync = self
5532 .state
5533 .base_sync
5534 .as_mut()
5535 .context("reserved synchronization has no lifetime accounting")?;
5536 sync.conflict = Some(why.clone());
5537 self.state.event("land", why);
5538 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5539 .await?;
5540 return Ok(());
5541 }
5542 if head != op.head {
5543 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5544 }
5545 tip = op.tip.clone();
5546 }
5547 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5548 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5549 if pending.is_none()
5550 && !conflicted
5551 && fetched_base
5552 && self
5553 .state
5554 .base_sync_recovery
5555 .episodes
5556 .last()
5557 .is_some_and(|e| e.boundary.is_some())
5558 && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5559 && let Ok(owner) = crate::queue::Queue::open().get(&task)
5560 && owner.execution_runs().last() == Some(&self.state.id)
5561 && owner.status != crate::queue::TaskStatus::Done
5562 && !owner.fresh_start
5563 && owner.review_branch.is_none()
5564 && crate::daemon::compatible_run_instruction(&self.state, &owner)
5565 && let Some(approval) = owner.resumption.as_ref()
5566 && approval.instruction == owner.instruction
5567 && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5568 && git::is_clean(&winner.worktree).await?
5569 && self.state.renew_base_episode(&task, approval, &head, &tip)
5570 {
5571 self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5572 self.state.save()?;
5573 }
5574
5575 if (behind > 0 || conflicted || head == tip)
5584 && self
5585 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5586 .await?
5587 {
5588 return Ok(());
5589 }
5590 if conflicted && self.state.base_recovery_pending() {
5591 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5592 let owner =
5593 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5594 let owner_valid = owner.as_ref().is_ok_and(|t| {
5595 t.execution_runs().last() == Some(&self.state.id)
5596 && t.status != crate::queue::TaskStatus::Done
5597 && !t.fresh_start
5598 && t.review_branch.is_none()
5599 });
5600 let invalidation = if !owner_valid {
5601 Some("latest task ownership changed")
5602 } else if !owner.as_ref().is_ok_and(|t| {
5603 authorization
5604 .generation
5605 .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5606 }) {
5607 Some("owner approval generation changed")
5608 } else if let Some(scope) = &authorization.scope {
5609 if scope.tip != tip {
5610 Some("fetched base changed")
5611 } else if !self
5612 .state
5613 .base_sync_recovery
5614 .episodes
5615 .last()
5616 .is_some_and(|e| e.id == scope.episode)
5617 {
5618 Some("active synchronization interval changed")
5619 } else if !owner
5620 .as_ref()
5621 .is_ok_and(|t| t.instruction == scope.instruction)
5622 {
5623 Some("task instruction changed")
5624 } else {
5625 None
5626 }
5627 } else {
5628 None
5629 };
5630 if let Some(reason) = invalidation {
5631 let authorization = self
5632 .state
5633 .base_sync_recovery
5634 .authorizations
5635 .last_mut()
5636 .unwrap();
5637 authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5638 self.state.save()?;
5639 return Ok(());
5640 }
5641 if !fetched_base
5642 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5643 || !git::is_clean(&winner.worktree).await?
5644 || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5645 {
5646 return Ok(());
5647 }
5648 }
5649 let recovery = conflicted && self.state.consume_base_recovery(&head);
5650 if conflicted {
5651 self.state.save()?;
5653 if !recovery {
5654 return Ok(());
5655 }
5656 }
5657
5658 if behind == 0 {
5659 if let Some(from) = self
5668 .state
5669 .rebase_fixes
5670 .iter()
5671 .rev()
5672 .find_map(|r| r.from.clone())
5673 && from != head
5674 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5675 .await
5676 .is_ok_and(|o| o.ok())
5677 {
5678 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5679 }
5680 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5686 if let Some(why) = &conflict {
5687 self.state.status = RunStatus::Blocked;
5688 self.state.event("land", why.clone());
5689 }
5690 self.state.base_sync = Some(BaseSync {
5691 tip,
5692 behind: 0,
5693 attempts,
5694 conflict,
5695 already_in: None,
5696 });
5697 if pending.is_some() {
5698 self.state.finish_base_operation(Some(&head));
5699 }
5700 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5701 .await?;
5702 self.state.save()?;
5703 return Ok(());
5704 }
5705
5706 if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5707 let why = format!(
5708 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5709 synchronization(s); synchronizing again would only race it",
5710 winner.branch
5711 );
5712 self.state.status = RunStatus::Blocked;
5713 self.state.base_sync = Some(BaseSync {
5714 tip,
5715 behind,
5716 attempts,
5717 conflict: Some(why.clone()),
5718 already_in: None,
5719 });
5720 self.state.event("land", why);
5721 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5722 .await?;
5723 self.state.save()?;
5724 return Ok(());
5725 }
5726
5727 if self.state.foreign_checkout.is_some() {
5728 let wait = repository_wait(&self.state, None);
5731 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5732 if self.state.base_sync.is_none() {
5733 self.state.base_sync = Some(BaseSync {
5734 tip: tip.clone(),
5735 behind,
5736 attempts: 0,
5737 conflict: None,
5738 already_in: None,
5739 });
5740 }
5741 self.state.reserve_base_operation(&head, &tip);
5742 self.state.save()?;
5743 let merged = git::git_raw(
5744 &winner.worktree,
5745 &[
5746 "-c",
5747 "core.hooksPath=/dev/null",
5748 "merge",
5749 "--no-ff",
5750 "--no-autostash",
5751 "--no-edit",
5752 "-m",
5753 &format!("Merge {tracking} before verifying"),
5754 &tip,
5755 ],
5756 )
5757 .await?;
5758 let conflict = if merged.ok() {
5759 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5760 self.state.event(
5761 "land",
5762 format!("merged {tracking} into {} before verifying", winner.branch),
5763 );
5764 None
5765 } else {
5766 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5767 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5768 }
5769 let why = format!(
5770 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5771 winner.branch,
5772 merged.stderr.chars().take(600).collect::<String>()
5773 );
5774 self.state.status = RunStatus::Blocked;
5775 self.state.event("land", &why);
5776 Some(why)
5777 };
5778 let synced = conflict.is_none();
5779 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5780 self.state
5781 .finish_base_operation(synced.then_some(fresh.as_str()));
5782 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5783 self.state.base_sync = Some(BaseSync {
5784 tip,
5785 behind: if synced { 0 } else { behind },
5786 attempts,
5787 conflict,
5788 already_in: None,
5789 });
5790 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5791 .await?;
5792 self.state.save()?;
5793 return Ok(());
5794 }
5795
5796 self.state.event(
5797 "land",
5798 format!(
5799 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5800 winner.branch
5801 ),
5802 );
5803 self.state.save()?;
5804
5805 let branch_tracking = format!("{remote}/{}", winner.branch);
5811 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5812 let remote_tip = if self.state.task_pr_branch().is_none()
5813 && matches!(&fetched_branch, Ok(o) if o.ok())
5814 {
5815 git::rev_parse(&repo, &branch_tracking).await.ok()
5816 } else {
5817 None
5818 };
5819 if let Some(theirs) = &remote_tip
5823 && !git::is_ancestor(&repo, theirs, &head).await
5824 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5825 .await
5826 .is_ok_and(|missing| missing.is_empty())
5827 {
5828 let why = format!(
5829 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5830 them",
5831 short(theirs),
5832 winner.branch
5833 );
5834 self.state.status = RunStatus::Blocked;
5835 self.state.base_sync = Some(BaseSync {
5836 tip,
5837 behind,
5838 attempts,
5839 conflict: Some(why.clone()),
5840 already_in: None,
5841 });
5842 self.state.event("land", why);
5843 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5844 .await?;
5845 self.state.save()?;
5846 return Ok(());
5847 }
5848
5849 if self.state.base_sync.is_none() {
5850 self.state.base_sync = Some(BaseSync {
5851 tip: tip.clone(),
5852 behind,
5853 attempts: 0,
5854 conflict: None,
5855 already_in: None,
5856 });
5857 }
5858 self.state.reserve_base_operation(&head, &tip);
5859 self.state.save()?;
5860 let scratch = self.state.dir().join("base-sync");
5861 let rebased =
5862 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5863 .await
5864 {
5865 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5866 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5867 Err(e) => Err(e),
5868 };
5869 let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5870 let stop_kind = match &rebased {
5871 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5872 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5873 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5874 };
5875 match rebased {
5876 Ok(None) => {
5877 crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5881 let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5882 self.state.finish_base_operation(Some(&fresh));
5883 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5884 let mut conflict = None;
5885 if let Some(pinned) = &remote_tip {
5886 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5887 match pushed {
5888 Ok(o) if o.ok() => self.state.event(
5889 "land",
5890 format!("pushed rebased {} to {remote}", winner.branch),
5891 ),
5892 Ok(o) => {
5893 conflict = Some(format!(
5894 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5895 winner.branch,
5896 short(pinned),
5897 o.stderr.chars().take(600).collect::<String>()
5898 ));
5899 }
5900 Err(e) => {
5901 conflict = Some(format!(
5902 "rebased {} locally but could not push it: {e:#}",
5903 winner.branch
5904 ));
5905 }
5906 }
5907 }
5908 if let Some(why) = &conflict {
5909 self.state.status = RunStatus::Blocked;
5910 self.state.event("land", why.clone());
5911 }
5912 self.state.base_sync = Some(BaseSync {
5913 tip: tip.clone(),
5914 behind: 0,
5915 attempts,
5916 conflict,
5917 already_in: None,
5918 });
5919 self.state
5920 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5921 }
5922 Ok(Some(conflict)) => {
5923 self.state.finish_base_operation(None);
5924 let why = format!(
5925 "{} conflicts with {tracking} and did not rebase: {}",
5926 winner.branch,
5927 conflict.chars().take(600).collect::<String>()
5928 );
5929 self.state.status = RunStatus::Blocked;
5930 self.state.base_sync = Some(BaseSync {
5931 tip,
5932 behind,
5933 attempts,
5934 conflict: Some(why.clone()),
5935 already_in: None,
5936 });
5937 self.state.event("land", why);
5938 }
5939 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5943 self.state.save()?;
5944 return Ok(());
5945 }
5946 Err(e) => {
5947 self.state.finish_base_operation(None);
5948 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5949 self.state.status = RunStatus::Blocked;
5950 self.state.base_sync = Some(BaseSync {
5951 tip,
5952 behind,
5953 attempts,
5954 conflict: Some(why.clone()),
5955 already_in: None,
5956 });
5957 self.state.event("land", why);
5958 }
5959 }
5960 if !self.state.base_sync_blocked() {
5961 self.state.base_sync_recovery.stop = None;
5962 }
5963 self.hold_base_sync_kind(stop_kind).await?;
5964 self.state.save()?;
5965 Ok(())
5966 }
5967
5968 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5974 if self.state.task_pr_branch().is_some() {
5977 return None;
5978 }
5979 let repo = self.state.repo.clone();
5980 let remote = self.state.config.merge.remote.clone();
5981 let fetched = git::fetch(&repo, &remote, branch).await;
5982 if !matches!(&fetched, Ok(o) if o.ok()) {
5983 return None;
5984 }
5985 let branch_tracking = format!("{remote}/{branch}");
5986 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5987 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5988 return None;
5989 }
5990 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5991 .await
5992 .is_ok_and(|missing| missing.is_empty())
5993 {
5994 return Some(format!(
5995 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5996 them",
5997 short(&theirs)
5998 ));
5999 }
6000 match git::push_pinned(&repo, &remote, branch, &theirs).await {
6001 Ok(o) if o.ok() => {
6002 self.state
6003 .event("land", format!("pushed rebased {branch} to {remote}"));
6004 None
6005 }
6006 Ok(o) => Some(format!(
6007 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6008 someone may have pushed): {}",
6009 short(&theirs),
6010 o.stderr.chars().take(600).collect::<String>()
6011 )),
6012 Err(e) => Some(format!(
6013 "{branch} is rebased locally but could not be pushed: {e:#}"
6014 )),
6015 }
6016 }
6017
6018 async fn settle_already_in(
6035 &mut self,
6036 branch: &str,
6037 tip: &str,
6038 head: &str,
6039 attempts: usize,
6040 behind: usize,
6041 ) -> Result<bool> {
6042 let repo = self.state.repo.clone();
6043 let remote = self.state.config.merge.remote.clone();
6044 let start = self.state.base_commit.clone();
6045 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6046 Ok(Some(e)) => e,
6047 Ok(None) => return Ok(false),
6048 Err(e) => {
6049 tracing::warn!("already-in-base check for {branch}: {e:#}");
6050 return Ok(false);
6051 }
6052 };
6053 let mut verified = vec![head.to_owned()];
6054 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6055 let fetched = git::fetch(&repo, &remote, &public_branch).await;
6056 if matches!(&fetched, Ok(o) if o.ok())
6057 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6058 && theirs != head
6059 {
6060 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6061 Ok(Some(_)) => verified.push(theirs),
6062 _ => return Ok(false),
6063 }
6064 }
6065 let base_branch = self.state.base_branch.clone();
6066 let message = format!(
6067 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6068 land",
6069 evidence.names(),
6070 evidence.proof.as_str()
6071 );
6072 let closed =
6073 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6074 .await;
6075 match closed {
6076 Ok(Ok(url)) => self
6077 .state
6078 .event("land", format!("closed {url}: superseded on {base_branch}")),
6079 Ok(Err(why)) => self
6080 .state
6081 .event("land", format!("did not close a pull request: {why}")),
6082 Err(e) => {
6083 let why = format!(
6084 "{branch} is already in {remote}/{base_branch}, but its pull request could \
6085 not be closed ({e:#}); resume to retry"
6086 );
6087 self.state.status = RunStatus::Blocked;
6088 self.state.base_sync = Some(BaseSync {
6089 tip: tip.to_owned(),
6090 behind,
6091 attempts,
6092 conflict: Some(why.clone()),
6093 already_in: None,
6094 });
6095 self.state.event("land", why);
6096 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6097 .await?;
6098 self.state.save()?;
6099 return Ok(true);
6100 }
6101 }
6102 self.state.status = RunStatus::AlreadyInBase;
6103 self.state.base_sync_recovery.stop = None;
6104 self.state.base_sync = Some(BaseSync {
6105 tip: tip.to_owned(),
6106 behind,
6107 attempts,
6108 conflict: None,
6109 already_in: Some(evidence),
6110 });
6111 self.state.event("land", message);
6112 self.state.save()?;
6113 self.settle_questions();
6114 Ok(true)
6115 }
6116
6117 fn landing_base(&self) -> String {
6127 self.state
6128 .base_sync
6129 .as_ref()
6130 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6131 }
6132
6133 pub async fn fix_selected(
6166 &mut self,
6167 ids: &[String],
6168 reason: &str,
6169 allow_stale: bool,
6170 ) -> Result<()> {
6171 let reason = reason.trim();
6172 if reason.is_empty() {
6173 bail!("a fix request needs a reason — that is the operator's own record of why");
6174 }
6175 if ids.is_empty() {
6176 bail!("no finding id given");
6177 }
6178 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6179 bail!(
6180 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6181 has already concluded — can be given a targeted fix. A run still \
6182 in progress should simply be resumed; a `merged` run's branch has \
6183 already landed, so its answer is a fresh `magi review <branch>`, \
6184 not reopening this run's own record",
6185 self.state.id,
6186 self.state.status.as_str()
6187 );
6188 }
6189 let _claim = FixClaim::acquire(&self.state.dir())?;
6192 crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6193 let Some(winner) = self.state.winner().cloned() else {
6194 bail!("run {} has no winning candidate to fix", self.state.id);
6195 };
6196 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6197 bail!(
6198 "branch `{}` no longer exists; this run cannot be extended",
6199 winner.branch
6200 );
6201 }
6202 let home = crate::run::home();
6203 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6204 bail!(
6205 "run {} is currently being worked on by another magi process",
6206 self.state.id
6207 );
6208 }
6209 let mut seen = BTreeSet::new();
6213 let mut findings = Vec::new();
6214 let mut missing = Vec::new();
6215 for id in ids {
6216 if !seen.insert(id.clone()) {
6217 continue;
6218 }
6219 match self.state.finding(id) {
6220 Some((round, rec, f)) => findings.push(OperatorFixFinding {
6221 id: f.id.clone(),
6222 severity: f.severity,
6223 reviewer_vote: rec.vote,
6224 round: round.round,
6225 round_head: round.head.clone(),
6226 reviewer: rec.reviewer,
6227 agent: rec.agent.clone(),
6228 file: f.file.clone(),
6229 line: f.line,
6230 title: f.title.clone(),
6231 detail: f.detail.clone(),
6232 outcome: OperatorFixOutcome::Pending,
6233 }),
6234 None => missing.push(id.clone()),
6235 }
6236 }
6237 if !missing.is_empty() {
6238 bail!(
6239 "unknown finding id(s): {}; nothing was changed",
6240 missing.join(", ")
6241 );
6242 }
6243
6244 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6245 let stale_details: Vec<(String, String)> = findings
6246 .iter()
6247 .filter(|f| f.round_head != head_at_request)
6248 .map(|f| (f.id.clone(), f.round_head.clone()))
6249 .collect();
6250 let stale = !stale_details.is_empty();
6251 if stale && !allow_stale {
6252 bail!(
6253 "the branch has moved since some finding(s) were raised — {} — now \
6254 at {}; pass --allow-stale to fix anyway, or re-run review first",
6255 stale_details
6256 .iter()
6257 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6258 .collect::<Vec<_>>()
6259 .join(", "),
6260 short(&head_at_request)
6261 );
6262 }
6263
6264 let request = OperatorFixRequest {
6265 requested_at: Timestamp::now(),
6266 reason: reason.to_owned(),
6267 findings,
6268 head_at_request: head_at_request.clone(),
6269 allow_stale,
6270 stale,
6271 fix: None,
6272 result_head: None,
6273 follow_up_review_run: None,
6274 };
6275 self.state.event(
6276 "fix",
6277 format!(
6278 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6279 request.findings.len(),
6280 request
6281 .findings
6282 .iter()
6283 .map(|f| f.id.as_str())
6284 .collect::<Vec<_>>()
6285 .join(", "),
6286 ),
6287 );
6288 self.state.operator_fixes.push(request);
6295 self.state.save()?;
6296 let request_index = self.state.operator_fixes.len() - 1;
6297
6298 let repo = self.state.repo.clone();
6307 let coordination = self.coordinate(&repo).await?;
6308 if winner.worktree.exists() {
6309 let dirty = git::git(
6312 &winner.worktree,
6313 &["status", "--porcelain", "--untracked-files=all"],
6314 )
6315 .await?;
6316 let only_withheld = dirty.lines().all(|l| {
6317 l.strip_prefix("?? ")
6318 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6319 });
6320 if !only_withheld {
6321 bail!(
6322 "`{}` has uncommitted changes; refusing to touch it — commit or \
6323 discard them first",
6324 winner.worktree.display()
6325 );
6326 }
6327 git::worktree_remove(&self.state.repo, &winner.worktree)
6328 .await
6329 .ok();
6330 }
6331 let fix_worktree = self.state.worktree_root().join("operator-fix");
6332 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6333 git::git(
6334 &self.state.repo,
6335 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6336 )
6337 .await
6338 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6339 if !git::is_clean(&fix_worktree).await? {
6340 git::worktree_remove(&self.state.repo, &fix_worktree)
6341 .await
6342 .ok();
6343 bail!(
6344 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6345 winner.branch
6346 );
6347 }
6348 let repo_for_setup = self.state.repo.clone();
6349 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6350 Ok(()) => {
6351 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6352 }
6353 Err(e) => Err(e),
6354 };
6355 if let Err(e) = setup {
6356 git::worktree_remove(&self.state.repo, &fix_worktree)
6357 .await
6358 .ok();
6359 return Err(e);
6360 }
6361
6362 self.state.resource_block = None;
6365
6366 let run_id = self.state.id.clone();
6367 let prompts = self.state.config.prompts.clone();
6368 let language = self.state.config.graph.language.clone();
6369 let sessions = self.state.config.graph.sessions;
6370 let artifacts = agent::artifacts_dir(&self.state.dir());
6371 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6372 .findings
6373 .iter()
6374 .map(|f| Finding {
6375 id: f.id.clone(),
6376 severity: f.severity,
6377 file: f.file.clone(),
6378 line: f.line,
6379 title: f.title.clone(),
6380 detail: f.detail.clone(),
6381 })
6382 .collect();
6383 let fix_prompt = prompt::operator_fix_in(
6384 &self.state.instruction,
6385 &finding_list,
6386 reason,
6387 &stale_details,
6388 &head_at_request,
6389 &language,
6390 &self.state.config.graph.github_language,
6391 );
6392 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6393 drop(coordination);
6394 let (job, seat, out) = self
6395 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6396 prompt: fix_prompt.clone(),
6397 spec,
6398 seat,
6399 cwd: fix_worktree.clone(),
6400 timeout,
6401 allow_write: true,
6402 sessions,
6403 artifacts: artifacts.clone(),
6404 stem: "operator-fix".to_owned(),
6405 handover: None,
6406 })
6407 .await;
6408 let agent_id = seat.agent.clone();
6409
6410 let mut fix = FixRecord {
6411 agent: agent_id,
6412 addressed: Vec::new(),
6413 rejected: Vec::new(),
6414 notes: String::new(),
6415 committed: false,
6416 failed: None,
6417 duration_ms: 0,
6418 continuation: None,
6419 };
6420 let mut final_seat = seat.clone();
6421 match out {
6422 AgentOutcome::Ok(o) => {
6423 fix.duration_ms = o.duration_ms;
6424 let parsed = verdict::extract_json::<FixReport>(&o.text);
6425 let incomplete_reason = match &parsed {
6426 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6427 "the reply parsed, but it reported a command whose own CLI \
6428 never confirmed an exit status"
6429 .to_owned(),
6430 ),
6431 Ok(_) => None,
6432 Err(e) => Some(e.to_string()),
6433 };
6434 match incomplete_reason {
6435 None => {
6436 let report = parsed.expect("checked Ok above");
6437 fix.addressed = report.addressed;
6438 fix.rejected = report.rejected;
6439 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6440 }
6441 Some(reason) => {
6442 let (resumed_seat, resolved, failure, cont) = self
6443 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6444 .await;
6445 fix.duration_ms += cont.cumulative_wait_ms;
6446 fix.continuation = Some(cont);
6447 final_seat = resumed_seat;
6448 match resolved {
6449 Some(report) => {
6450 fix.addressed = report.addressed;
6451 fix.rejected = report.rejected;
6452 fix.notes =
6453 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6454 }
6455 None => fix.failed = failure,
6456 }
6457 }
6458 }
6459 }
6460 AgentOutcome::Dropped(o) => {
6461 fix.duration_ms = o.duration_ms;
6462 let why = o
6463 .dropped
6464 .as_ref()
6465 .map(|d| d.why.as_str())
6466 .unwrap_or("the CLI ended the stream without delivering its answer");
6467 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6468 }
6469 AgentOutcome::Quota(o) => {
6470 self.state.quota.push(QuotaLoss::from_report(
6471 final_seat.key.clone(),
6472 "fix",
6473 o.quota.as_ref(),
6474 ));
6475 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6476 }
6477 AgentOutcome::Failed(e) => fix.failed = Some(e),
6478 AgentOutcome::ResourceBlocked(e) => {
6479 fix.failed = Some(format!("the fixer was not started: {e}"));
6480 }
6481 }
6482 if fix.continuation.is_none() {
6485 fix.continuation = Some(ContinuationRecord::not_needed());
6486 }
6487 self.state.seats.insert(final_seat.key.clone(), final_seat);
6488
6489 let rescue_message = format!(
6490 "magi: operator-selected fix ({}) (uncommitted work)",
6491 self.state.operator_fixes[request_index]
6492 .findings
6493 .iter()
6494 .map(|f| f.id.as_str())
6495 .collect::<Vec<_>>()
6496 .join(", ")
6497 );
6498 let repo = self.state.repo.clone();
6499 let coordination = self.coordinate(&repo).await?;
6500 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6501 self.state.note_withheld("fix", &r.withheld);
6502 }
6503 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6504 fix.committed = after != head_at_request;
6505 git::worktree_remove(&self.state.repo, &fix_worktree)
6506 .await
6507 .ok();
6508 drop(coordination);
6509
6510 self.state.event(
6511 "fix",
6512 match &fix.failed {
6513 Some(reason) => format!(
6514 "operator fix: adoption report was lost ({reason}); {}",
6515 if fix.committed {
6516 "committed"
6517 } else {
6518 "NO new commit"
6519 }
6520 ),
6521 None => format!(
6522 "operator fix: {} addressed, {} rejected, {}",
6523 fix.addressed.len(),
6524 fix.rejected.len(),
6525 if fix.committed {
6526 "committed"
6527 } else {
6528 "NO new commit"
6529 }
6530 ),
6531 },
6532 );
6533
6534 for f in &mut self.state.operator_fixes[request_index].findings {
6541 f.outcome = if fix.failed.is_some() {
6542 OperatorFixOutcome::Unreported
6543 } else if fix.addressed.contains(&f.id) {
6544 OperatorFixOutcome::Addressed
6545 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6546 OperatorFixOutcome::Rejected { why: r.why.clone() }
6547 } else {
6548 OperatorFixOutcome::Unreported
6549 };
6550 }
6551
6552 let committed = fix.committed;
6553 if committed {
6554 self.state.operator_fixes[request_index].result_head = Some(after.clone());
6555 }
6556 self.state.operator_fixes[request_index].fix = Some(fix);
6557 self.state.save()?;
6560 if self.state.resource_block.is_some() {
6561 return self.stop_for_resource();
6562 }
6563
6564 if committed {
6565 self.state.event(
6566 "fix",
6567 format!(
6568 "operator fix committed {}; opening a follow-up review-only run",
6569 short(&after)
6570 ),
6571 );
6572 let origin =
6575 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6576 match Self::review(
6577 &self.state.repo,
6578 &winner.branch,
6579 self.state.config.clone(),
6580 origin,
6581 )
6582 .await
6583 {
6584 Ok(mut follow_up) => {
6585 follow_up.state.event(
6586 "start",
6587 format!(
6588 "requested by an operator fix on run {} for finding(s) {}",
6589 self.state.id,
6590 self.state.operator_fixes[request_index]
6591 .findings
6592 .iter()
6593 .map(|f| f.id.as_str())
6594 .collect::<Vec<_>>()
6595 .join(", "),
6596 ),
6597 );
6598 follow_up.state.save()?;
6599 let follow_up_id = follow_up.state.id.clone();
6600 let adopted = match crate::direct::adopt(
6603 &follow_up.state,
6604 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6605 ) {
6606 Ok(a) => a,
6607 Err(e) => {
6608 self.state.event(
6611 "fix",
6612 format!(
6613 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6614 ),
6615 );
6616 self.state.save()?;
6617 return Ok(());
6618 }
6619 };
6620 let executed = follow_up.execute().await;
6621 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6622 if let Some(a) = adopted {
6623 a.finish(&follow_up.state, executed);
6624 }
6625 if let Some(e) = failure {
6626 self.state.event(
6627 "fix",
6628 format!(
6629 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6630 ),
6631 );
6632 }
6633 self.state.operator_fixes[request_index].follow_up_review_run =
6634 Some(follow_up_id);
6635 }
6636 Err(e) => {
6637 self.state.event(
6638 "fix",
6639 format!("committed the fix but could not open a follow-up review: {e:#}"),
6640 );
6641 }
6642 }
6643 self.state.save()?;
6644 }
6645
6646 Ok(())
6647 }
6648
6649 async fn ask_fixer(
6669 &mut self,
6670 winner: &Candidate,
6671 node: &'static str,
6672 round: Option<usize>,
6673 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6674 ) -> (SeatJob, SeatState, AgentOutcome) {
6675 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6676 self.ask_fixer_among(attempts, node, round, None, build)
6677 .await
6678 }
6679
6680 async fn ask_fixer_among(
6685 &mut self,
6686 attempts: Vec<(AgentSpec, String)>,
6687 node: &'static str,
6688 round: Option<usize>,
6689 restore: Option<(&Path, &str)>,
6690 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6691 ) -> (SeatJob, SeatState, AgentOutcome) {
6692 let run_id = self.state.id.clone();
6693 let prompts = self.state.config.prompts.clone();
6694 let cache = self.state.config.cache_dir();
6695 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6696 let mut last = None;
6697 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6698 let seat = self.seat(&key, &spec.id);
6699 let mut job = build(spec.clone(), seat);
6700 if i > 0 {
6701 job.stem = format!("{}-{}", job.stem, spec.id);
6702 }
6703 let ctx = WaveCtx {
6704 carry_seats: false,
6705 pauses: Some((&self.pause, &self.interrupt)),
6706 run: &run_id,
6707 node,
6708 prompts: &prompts,
6709 cache: cache.as_deref(),
6710 round,
6711 };
6712 let (seat, out) =
6713 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6714 let advances = match &out {
6717 AgentOutcome::Ok(o) => agent::output_advances(o),
6718 AgentOutcome::ResourceBlocked(_) => false,
6719 _ => true,
6720 };
6721 if let Some(next) = ids.get(i + 1)
6722 && advances
6723 {
6724 self.state.seats.insert(seat.key.clone(), seat.clone());
6725 let class =
6726 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6727 let reason = match &out {
6728 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6729 other => fail_reason(other),
6730 };
6731 if let Some((wt, rev)) = restore
6735 && let Err(e) = self.graft_restore(wt, rev).await
6736 {
6737 let why =
6738 format!("could not restore the worktree before the next fixer: {e:#}");
6739 tracing::warn!("{why}");
6740 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6741 AgentOutcome::ResourceBlocked(why)
6742 } else {
6743 AgentOutcome::Failed(why)
6744 };
6745 last = Some((job, seat, out));
6746 break;
6747 }
6748 record_handover(
6749 &mut self.state,
6750 node,
6751 &seat.key,
6752 &spec.id,
6753 next,
6754 &class,
6755 &reason,
6756 );
6757 continue;
6758 }
6759 last = Some((job, seat, out));
6760 break;
6761 }
6762 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6763 if let AgentOutcome::Quota(output) = &out
6764 && node != "graft"
6765 {
6766 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6771 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6772 self.state.quota.push(loss);
6773 self.state.seats.insert(seat.key.clone(), seat.clone());
6774 self.state.status = RunStatus::Blocked;
6775 return (
6776 job,
6777 seat,
6778 AgentOutcome::ResourceBlocked(
6779 "fixer unavailable; restore agent availability and resume".into(),
6780 ),
6781 );
6782 }
6783 (job, seat, out)
6784 }
6785
6786 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6797 use crate::recovery::{Mode, Repair};
6798 if !self.state.reviews.is_empty() {
6799 return Ok(());
6800 }
6801 let Some(mut rec) = self.state.recovery.clone() else {
6802 return Ok(());
6803 };
6804 if rec.mode != Mode::VerifyRepair {
6805 return Ok(());
6806 }
6807 let wt = winner.worktree.clone();
6808 let mut head = git::rev_parse(&wt, "HEAD").await?;
6809 match rec.repair.clone() {
6810 Repair::Done { .. } => return Ok(()),
6811 Repair::WaitingForRestore { before } => {
6812 self.graft_restore(&wt, &before).await?;
6813 head = before;
6814 rec.repair = Repair::NotStarted;
6815 self.state.recovery = Some(rec.clone());
6816 self.state.save()?;
6817 }
6818 Repair::Started { before } => {
6819 if let Err(e) = self.graft_restore(&wt, &before).await {
6823 if e.is::<crate::repo_lock::ResourceBlocked>() {
6824 return Err(e);
6825 }
6826 return self
6827 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6828 .await;
6829 }
6830 rec.repair = Repair::Done {
6831 before: before.clone(),
6832 after: before,
6833 committed: false,
6834 failed: Some("interrupted; rolled back".to_owned()),
6835 };
6836 self.state
6837 .event("recovery", "the repair was interrupted and rolled back");
6838 self.state.recovery = Some(rec);
6839 self.state.save()?;
6840 return Ok(());
6841 }
6842 Repair::NotStarted => {}
6843 }
6844 if crate::recovery::recheck(&mut rec, &head) {
6845 self.state.event("recovery", rec.summary());
6846 self.state.recovery = Some(rec);
6847 self.state.save()?;
6848 return Ok(());
6849 }
6850 rec.repair = Repair::Started {
6851 before: head.clone(),
6852 };
6853 self.state.recovery = Some(rec.clone());
6854 self.state.save()?;
6855
6856 let language = self.state.config.graph.language.clone();
6857 let sessions = self.state.config.graph.sessions;
6858 let artifacts = agent::artifacts_dir(&self.state.dir());
6859 let fix_prompt = prompt::recovery_fix(
6860 &self.state.instruction,
6861 &rec,
6862 self.state.config.graph.review_rounds,
6863 &language,
6864 );
6865 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6866 self.state.event(
6867 "recovery",
6868 format!(
6869 "repair: the fixer starts from the failed verification on {}",
6870 short(&head)
6871 ),
6872 );
6873 let (_job, seat, out) = self
6874 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6875 prompt: fix_prompt.clone(),
6876 spec,
6877 seat,
6878 cwd: wt.clone(),
6879 timeout,
6880 allow_write: true,
6881 sessions,
6882 artifacts: artifacts.clone(),
6883 stem: "fix-recovery".to_owned(),
6884 handover: None,
6885 })
6886 .await;
6887 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6888 rec.repair = Repair::WaitingForRestore { before: head };
6889 self.state.recovery = Some(rec);
6890 return self.stop_for_resource();
6891 }
6892 let failed = match &out {
6893 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6894 AgentOutcome::Quota(o) => {
6895 self.state.quota.push(QuotaLoss::from_report(
6896 seat.key.clone(),
6897 "fix",
6898 o.quota.as_ref(),
6899 ));
6900 Some("rate limited (quota); the fixer could not run".to_owned())
6901 }
6902 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6903 AgentOutcome::Failed(e) => Some(e.clone()),
6904 };
6905 self.state.seats.insert(seat.key.clone(), seat);
6906 let coordination = self.coordinate(&wt).await?;
6907 let failed = match failed {
6908 Some(why) => {
6910 if let Err(e) = git::restore_to(&wt, &head).await {
6911 return self
6912 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6913 .await;
6914 }
6915 Some(why)
6916 }
6917 None => {
6918 if let Ok(r) = git::rescue_commit(
6919 &wt,
6920 "magi: recovery repair of the failed verification (uncommitted work)",
6921 )
6922 .await
6923 {
6924 self.state.note_withheld("fix", &r.withheld);
6925 }
6926 None
6927 }
6928 };
6929 let after = git::rev_parse(&wt, "HEAD").await?;
6930 drop(coordination);
6931 self.state.event(
6932 "recovery",
6933 match &failed {
6934 Some(why) => format!("repair: nothing kept ({why})"),
6935 None if after != head => {
6936 format!("repair: committed {}", short(&after))
6937 }
6938 None => "repair: the fixer made no commit".to_owned(),
6939 },
6940 );
6941 rec.repair = Repair::Done {
6942 before: head.clone(),
6943 committed: after != head,
6944 after,
6945 failed,
6946 };
6947 self.state.recovery = Some(rec);
6948 self.state.save()?;
6949 Ok(())
6950 }
6951
6952 async fn stop_recovery(
6955 &mut self,
6956 mut rec: crate::recovery::RecoveryRecord,
6957 head: &str,
6958 why: String,
6959 ) -> Result<()> {
6960 rec.repair = crate::recovery::Repair::Started {
6962 before: head.to_owned(),
6963 };
6964 self.state.recovery = Some(rec);
6965 self.state
6966 .event("recovery", format!("repair could not be undone: {why}"));
6967 self.state.status = RunStatus::Blocked;
6968 self.state.save()?;
6969 bail!("recovery repair could not be rolled back: {why}")
6970 }
6971
6972 async fn review_loop(&mut self) -> Result<()> {
6973 if self.hold_base_sync().await? {
6974 return Ok(());
6975 }
6976 loop {
6977 if self
6978 .state
6979 .base_sync
6980 .as_ref()
6981 .is_some_and(|s| s.conflict.is_some())
6982 {
6983 return Ok(());
6984 }
6985 if self.catchup_repairable() {
6986 let record = self.state.reviews.last().expect("repairable round").clone();
6987 let winner = self
6988 .state
6989 .winner()
6990 .cloned()
6991 .context("catch-up repair has no winner")?;
6992 if !self.fix_review_round(&winner, record).await? {
6993 return Ok(());
6994 }
6995 }
6996 self.review_loop_pass().await?;
6997 if !self.catchup_repairable() {
6998 return Ok(());
6999 }
7000 }
7001 }
7002
7003 fn catchup_repairable(&self) -> bool {
7006 self.state.reviews.len() < self.state.config.graph.review_rounds
7007 && self.state.reviews.last().is_some_and(|r| {
7008 !r.clean
7009 && r.blocking == 0
7010 && r.fix.is_none()
7011 && r.e2e_status() == E2eStatus::Failed
7012 })
7013 }
7014
7015 async fn review_loop_pass(&mut self) -> Result<()> {
7016 if self
7021 .state
7022 .base_sync
7023 .as_ref()
7024 .is_some_and(|s| s.conflict.is_some())
7025 {
7026 return Ok(());
7027 }
7028 let run_id = self.state.id.clone();
7033 let prompts = self.state.config.prompts.clone();
7034 let Some(winner) = self.state.winner().cloned() else {
7035 return Ok(());
7036 };
7037 let granted_review = self
7038 .state
7039 .repair_grants
7040 .last()
7041 .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7042 .cloned();
7043 let fresh_review = granted_review
7044 .as_ref()
7045 .is_some_and(|g| self.state.reviews.len() == g.review_start);
7046 let max_rounds = granted_review
7047 .as_ref()
7048 .map_or(self.state.config.graph.review_rounds, |g| {
7049 g.review_start.saturating_add(1)
7050 });
7051 if let Some(status) =
7061 review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7062 {
7063 if status == RunStatus::Gating {
7066 self.record_contested_handoff();
7067 }
7068 self.state.status = status;
7069 self.state.save()?;
7070 return Ok(());
7071 }
7072 self.state.status = RunStatus::Reviewing;
7073 if !fresh_review
7083 && self.state.reviews.last().is_some_and(|r| {
7084 r.e2e_status() == E2eStatus::ResourceBlocked
7085 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7086 })
7087 {
7088 let shell = self.state.config.shell();
7089 return self
7090 .stop_unresolved_review(
7091 "the last round's own verification never resolved",
7092 &shell,
7093 &winner.worktree,
7094 )
7095 .await;
7096 }
7097
7098 if granted_review.is_none() {
7099 self.recovery_repair(&winner).await?;
7100 }
7101 if self.resource_halt()? {
7102 return Ok(());
7103 }
7104
7105 let repo = self.state.repo.clone();
7106 let root = self.state.worktree_root();
7107 let language = self.state.config.graph.language.clone();
7108 let sessions = self.state.config.graph.sessions;
7109 let artifacts = self.merge_repair_artifacts();
7110 let base = self.landing_base();
7111 let base_short = short(&base);
7112 let reviewers = self.roles.reviewers.clone();
7113 let shell = self.state.config.shell();
7114
7115 for round in (self.state.reviews.len() + 1)..=max_rounds {
7116 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7117 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7118 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7119 let prev_verification = self
7128 .state
7129 .reviews
7130 .last()
7131 .and_then(|r| r.verification_summary(&head));
7132 let prior_section = match (&self.state.recovery, round) {
7135 (Some(rec), 1) if self.state.reviews.is_empty() => {
7136 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7137 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7138 Some(crate::recovery::head_of(
7139 &git::diff(&winner.worktree, reviewed, "HEAD")
7140 .await
7141 .unwrap_or_default(),
7142 crate::recovery::SINCE_DIFF_MAX,
7143 ))
7144 }
7145 _ => None,
7146 };
7147 Some(prompt::recovery_review(rec, since.as_deref()))
7148 }
7149 _ => None,
7150 };
7151
7152 let mut jobs = Vec::new();
7156 let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7157 self.coordinate(&repo).await?
7158 } else {
7159 None
7160 };
7161 for (r, spec) in reviewers.iter().cloned().enumerate() {
7162 let wt = root.join(format!("review-{}", r + 1));
7163 if wt.exists() {
7166 crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7167 } else {
7168 let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7169 None
7170 } else {
7171 self.coordinate(&repo).await?
7172 };
7173 git::worktree_add_detached(&repo, &wt, &head).await?;
7174 }
7175 let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7176 None
7177 } else {
7178 self.coordinate(&repo).await?
7179 };
7180 self.setup_seat_worktree(&repo, &wt, false).await?;
7181 let seat_key = format!("review-{}", r + 1);
7182 let spec = pick_start_spec(
7185 &self.roles.reviewer_roster,
7186 spec,
7187 self.state.seat_history.get(&seat_key),
7188 );
7189 let seat = self.seat(&seat_key, &spec.id);
7190 jobs.push(SeatJob {
7191 prompt: prompt::review(&prompt::ReviewCtx {
7192 instruction: &self.state.instruction,
7193 branch: &winner.branch,
7194 base_short: &base_short,
7195 stat: &stat,
7196 patch: &patch,
7197 verification: prev_verification.as_ref(),
7198 prior: prior_section.as_deref(),
7199 reviewers: reviewers.len(),
7200 round,
7201 rounds: max_rounds,
7202 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7205 lens: Lens::for_seat(r),
7206 language: &language,
7207 github_language: &self.state.config.graph.github_language,
7208 }),
7209 spec,
7210 seat,
7211 cwd: wt,
7212 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7213 allow_write: false,
7214 sessions,
7215 artifacts: artifacts.clone(),
7216 stem: format!("review-{round}-{}", r + 1),
7217 handover: None,
7218 });
7219 }
7220
7221 drop(coordination);
7222 self.state.event(
7223 "review",
7224 format!(
7225 "round {round}: {} reviewers on {}",
7226 jobs.len(),
7227 short(&head)
7228 ),
7229 );
7230 let mut quota_losses = Vec::new();
7231 let review_retries = self.state.config.graph.retries;
7232 let review_cache = self.state.config.cache_dir();
7233 let ctx = WaveCtx {
7234 carry_seats: true,
7235 pauses: Some((&self.pause, &self.interrupt)),
7236 run: &run_id,
7237 node: "review",
7238 prompts: &prompts,
7239 cache: review_cache.as_deref(),
7240 round: Some(round),
7241 };
7242 let results = ask_json_wave::<Review>(
7243 jobs,
7244 Arc::clone(&self.sem),
7245 review_retries,
7246 &self.roles.reviewer_roster,
7247 &ctx,
7248 &mut quota_losses,
7249 &mut self.state,
7250 &|_: &Review| Ok(()),
7251 )
7252 .await;
7253 if self.state.resource_block.is_some() {
7254 return self.stop_for_resource();
7255 }
7256 let round_quota_missing = quota_losses.len();
7260 self.state.quota.extend(quota_losses);
7261
7262 let mut records = Vec::new();
7263 let mut all_findings = Vec::new();
7264 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7265 let agent_id = seat.agent.clone();
7266 self.state.seats.insert(seat.key.clone(), seat);
7267 let mut record = ReviewRecord {
7268 reviewer: r + 1,
7269 agent: agent_id,
7270 summary: String::new(),
7271 findings: Vec::new(),
7272 vote: None,
7273 failed: None,
7274 duration_ms: 0,
7275 attempts,
7281 };
7282 match res {
7283 Ok((review, out)) => {
7284 record.summary =
7292 blind::sanitize_prose(&review.summary, &self.state.config.blind);
7293 record.vote = Some(review.vote);
7294 record.duration_ms = out.duration_ms;
7295 for (n, mut f) in review.findings.into_iter().enumerate() {
7296 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7299 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7300 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7301 f.file = f
7307 .file
7308 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7309 all_findings.push(f.clone());
7310 record.findings.push(f);
7311 }
7312 self.state.event(
7313 "review",
7314 format!(
7315 "round {round}: reviewer {} voted {} with {} finding(s)",
7316 r + 1,
7317 review.vote.label(),
7318 record.findings.len()
7319 ),
7320 );
7321 }
7322 Err(e) => {
7323 record.failed = Some(e.to_string());
7324 self.state.event(
7325 "review",
7326 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7327 );
7328 }
7329 }
7330 records.push(record);
7331 }
7332
7333 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7340 let vote_split =
7341 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7342 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7343 if vote_split {
7344 self.state.event(
7345 "review",
7346 format!(
7347 "round {round}: votes split ({}) — one round of reconsideration",
7348 initial_votes
7349 .iter()
7350 .map(|v| v.label())
7351 .collect::<Vec<_>>()
7352 .join(", ")
7353 ),
7354 );
7355 let panel: Vec<ReviewSeatReport<'_>> = records
7358 .iter()
7359 .filter_map(|r| {
7360 r.vote.map(|vote| ReviewSeatReport {
7361 reviewer: r.reviewer,
7362 vote,
7363 summary: &r.summary,
7364 findings: &r.findings,
7365 })
7366 })
7367 .collect();
7368
7369 let mut jobs = Vec::new();
7370 let mut seats_at = Vec::new();
7371 for (r, spec) in reviewers.iter().cloned().enumerate() {
7372 if records[r].vote.is_none() {
7376 continue;
7377 }
7378 let wt = root.join(format!("review-{}", r + 1));
7379 let seat_key = format!("review-{}", r + 1);
7380 let spec = self.occupant(&seat_key, spec);
7381 let seat = self.seat(&seat_key, &spec.id);
7382 let build = |with_patch: bool| {
7390 prompt::review_reconsider(&ReviewReconsiderCtx {
7391 instruction: &self.state.instruction,
7392 reviewer: r + 1,
7393 lens: Lens::for_seat(r),
7394 panel: &panel,
7395 patch: with_patch.then_some(ReviewPatch {
7396 branch: &winner.branch,
7397 base_short: &base_short,
7398 stat: &stat,
7399 patch: &patch,
7400 }),
7401 round,
7402 rounds: max_rounds,
7403 language: &language,
7404 })
7405 };
7406 let full = build(true);
7407 let prompt = if has_context(&spec, &seat, sessions) {
7408 build(false)
7409 } else {
7410 full.clone()
7411 };
7412 jobs.push(SeatJob {
7413 prompt,
7414 spec,
7415 seat,
7416 cwd: wt,
7417 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7418 allow_write: false,
7419 sessions,
7420 artifacts: artifacts.clone(),
7421 stem: format!("review-{round}-reconsider-{}", r + 1),
7422 handover: Some(full),
7423 });
7424 seats_at.push(r);
7425 }
7426
7427 let mut recon_quota_losses = Vec::new();
7428 let recon_cache = self.state.config.cache_dir();
7429 let recon_ctx = WaveCtx {
7430 carry_seats: true,
7431 pauses: Some((&self.pause, &self.interrupt)),
7432 run: &run_id,
7433 node: "review",
7434 prompts: &prompts,
7435 cache: recon_cache.as_deref(),
7436 round: Some(round),
7437 };
7438 let recon_results = ask_json_wave::<ReviewRevote>(
7439 jobs,
7440 Arc::clone(&self.sem),
7441 review_retries,
7442 &self.roles.reviewer_roster,
7443 &recon_ctx,
7444 &mut recon_quota_losses,
7445 &mut self.state,
7446 &|_: &ReviewRevote| Ok(()),
7447 )
7448 .await;
7449 if self.state.resource_block.is_some() {
7450 return self.stop_for_resource();
7451 }
7452 self.state.quota.extend(recon_quota_losses);
7453
7454 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7455 let agent_id = seat.agent.clone();
7456 self.state.seats.insert(seat.key.clone(), seat);
7457 let mut rec = ReviewRevoteRecord {
7458 reviewer: r + 1,
7459 agent: agent_id,
7460 vote: None,
7461 reason: String::new(),
7462 failed: None,
7463 };
7464 match res {
7465 Ok((rv, _)) => {
7466 rec.vote = Some(rv.vote);
7467 rec.reason =
7468 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7469 self.state.event(
7470 "review",
7471 format!(
7472 "round {round}: reviewer {} revoted {}",
7473 r + 1,
7474 rv.vote.label()
7475 ),
7476 );
7477 }
7478 Err(e) => {
7479 rec.failed = Some(e.to_string());
7480 self.state.event(
7481 "review",
7482 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7483 );
7484 }
7485 }
7486 reconsideration.push(rec);
7487 }
7488 } else if initial_votes.len() > 1 {
7489 self.state.event(
7490 "review",
7491 format!(
7492 "round {round}: votes agreed ({}) — no reconsideration",
7493 initial_votes[0].label()
7494 ),
7495 );
7496 }
7497
7498 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7499 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7500 let defer_e2e =
7511 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7512 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7513 let reason =
7514 format!("{blocking} blocking finding(s) already required a fix this round");
7515 self.state.event(
7516 "verify",
7517 format!(
7518 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7519 {}); it will run once a round has none left",
7520 short(&head)
7521 ),
7522 );
7523 (Vec::new(), false, true, Some(reason))
7524 } else {
7525 let e2e_commands = self.state.config.verify.e2e.clone();
7526 let cache_dir = self.state.config.cache_dir();
7527 let context = format!("round {round}");
7528 let wait = self.repository_wait();
7529 let (e2e, verify_retried) = with_cache_lease(
7530 &mut self.state,
7531 cache_dir.as_deref(),
7532 "e2e",
7533 "e2e",
7534 &winner.worktree,
7535 &head,
7536 verify_timeout,
7537 &context,
7538 &wait,
7539 |state, budget| {
7540 let shell = shell.clone();
7541 let e2e_commands = e2e_commands.clone();
7542 let worktree = winner.worktree.clone();
7543 let context = context.clone();
7544 async move {
7545 run_e2e_with_retry(
7546 state,
7547 &shell,
7548 &e2e_commands,
7549 &worktree,
7550 budget,
7551 &context,
7552 )
7553 .await
7554 }
7555 },
7556 )
7557 .await?;
7558 (e2e, verify_retried, false, None)
7559 };
7560
7561 let expected = records.len();
7562 let answered = records.iter().filter(|r| r.failed.is_none()).count();
7563 let incomplete = answered < expected;
7564 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7565 let policy = self.state.config.graph.incomplete_review;
7566 let clean = round_is_clean(
7567 blocking,
7568 e2e_ok,
7569 answered,
7570 expected,
7571 round_quota_missing,
7572 policy,
7573 );
7574
7575 let mut round_record = ReviewRound {
7576 round,
7577 head: head.clone(),
7578 verified_head: None,
7579 verified_at: None,
7580 reviews: records,
7581 e2e,
7582 verify_retried,
7583 e2e_deferred,
7584 e2e_defer_reason,
7585 fix: None,
7586 blocking,
7587 answered,
7588 expected,
7589 clean,
7590 progressed: false,
7591 vote_split,
7592 reconsideration,
7593 verdict: None,
7594 };
7595 round_record.verdict = ReviewVote::worst(
7599 round_record
7600 .final_votes()
7601 .into_iter()
7602 .map(|(_, _, vote)| vote),
7603 );
7604 if !matches!(
7615 round_record.e2e_status(),
7616 E2eStatus::Deferred | E2eStatus::NotConfigured
7617 ) {
7618 round_record.verified_head = Some(head.clone());
7619 round_record.verified_at = Some(Timestamp::now());
7620 }
7621
7622 if incomplete {
7623 let missing: Vec<String> = round_record
7624 .reviews
7625 .iter()
7626 .filter(|r| r.failed.is_some())
7627 .map(|r| format!("review-{}", r.reviewer))
7628 .collect();
7629 self.state.event(
7630 "review",
7631 format!(
7632 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7633 missing.join(", ")
7634 ),
7635 );
7636 }
7637
7638 if clean {
7639 self.state.event(
7640 "review",
7641 if incomplete && policy == IncompleteReviewPolicy::Warn {
7642 format!(
7643 "round {round}: clean (warn policy, incomplete panel) — no \
7644 blocking findings from the seats that answered, verification green"
7645 )
7646 } else if incomplete {
7647 format!(
7648 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7649 quorum) — no blocking findings from the seats that answered, \
7650 verification green",
7651 expected - answered
7652 )
7653 } else {
7654 format!("round {round}: clean — no blocking findings, verification green")
7655 },
7656 );
7657 self.state.reviews.push(round_record);
7658 self.state.status = RunStatus::Gating;
7659 self.state.save()?;
7660 return Ok(());
7661 }
7662
7663 if incomplete && blocking == 0 && e2e_ok {
7671 self.state.reviews.push(round_record);
7672 self.state.save()?;
7673 if round == max_rounds {
7674 self.state.status = RunStatus::Blocked;
7675 self.state.event(
7676 "review",
7677 format!(
7678 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7679 refusing to call it clean",
7680 expected - answered
7681 ),
7682 );
7683 return Ok(());
7684 }
7685 continue;
7686 }
7687
7688 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7700 self.state.reviews.push(round_record);
7701 return self
7702 .stop_unresolved_review(
7703 "the round's own verification could not run",
7704 &shell,
7705 &winner.worktree,
7706 )
7707 .await;
7708 }
7709
7710 if round == max_rounds {
7711 self.state.reviews.push(round_record);
7712 return self
7713 .stop_reviewing(
7714 &format!(
7715 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7716 ),
7717 &shell,
7718 &winner.worktree,
7719 )
7720 .await;
7721 }
7722
7723 if !self.fix_review_round(&winner, round_record).await? {
7724 return Ok(());
7725 }
7726 }
7727 Ok(())
7728 }
7729
7730 async fn fix_review_round(
7733 &mut self,
7734 winner: &Candidate,
7735 mut round_record: ReviewRound,
7736 ) -> Result<bool> {
7737 let round = round_record.round;
7738 let shell = self.state.config.shell();
7739 let step = self.run_fix_round(winner, &mut round_record).await?;
7740 if matches!(step, FixStep::Resource) {
7741 self.stop_for_resource()?;
7742 return Ok(false);
7743 }
7744 if self.state.reviews.len() < round {
7745 self.state.reviews.push(round_record);
7746 } else {
7747 self.state.reviews[round - 1] = round_record;
7748 }
7749 self.state.save()?;
7750
7751 if matches!(step, FixStep::Recorded { report_lost: true }) {
7764 self.stop_reviewing(
7765 "the fixer's adoption report never came back, even after resuming its \
7766 own seat; refusing to start another round against the same worktree \
7767 while that is unresolved",
7768 &shell,
7769 &winner.worktree,
7770 )
7771 .await?;
7772 return Ok(false);
7773 }
7774
7775 let streak = self
7776 .state
7777 .reviews
7778 .iter()
7779 .rev()
7780 .take_while(|r| !r.progressed)
7781 .count();
7782 if streak >= STAGNANT_LIMIT {
7783 self.stop_reviewing(
7784 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7785 &shell,
7786 &winner.worktree,
7787 )
7788 .await?;
7789 return Ok(false);
7790 }
7791 Ok(true)
7792 }
7793
7794 async fn run_fix_round(
7796 &mut self,
7797 winner: &Candidate,
7798 round_record: &mut ReviewRound,
7799 ) -> Result<FixStep> {
7800 let round = round_record.round;
7801 let max_rounds = self.state.config.graph.review_rounds;
7802 let run_id = self.state.id.clone();
7803 let prompts = self.state.config.prompts.clone();
7804 let language = self.state.config.graph.language.clone();
7805 let sessions = self.state.config.graph.sessions;
7806 let artifacts = self.merge_repair_artifacts();
7807 let base = self.landing_base();
7808 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7809 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7810 let this_round_verification = round_record.verification_summary(&head);
7811 let all_findings: Vec<_> = round_record
7812 .reviews
7813 .iter()
7814 .flat_map(|r| r.findings.iter().cloned())
7815 .collect();
7816 let blocking_findings: Vec<_> = all_findings
7819 .iter()
7820 .filter(|f| f.severity.blocks())
7821 .cloned()
7822 .collect();
7823 let fix_prompt = prompt::fix_in(
7824 &self.state.instruction,
7825 &blocking_findings,
7826 this_round_verification.as_ref(),
7827 round,
7828 max_rounds,
7829 &language,
7830 &self.state.config.graph.github_language,
7831 );
7832 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7833 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7834 let (job, seat, out) = self
7835 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7836 prompt: fix_prompt.clone(),
7837 spec,
7838 seat,
7839 cwd: winner.worktree.clone(),
7840 timeout,
7841 allow_write: true,
7842 sessions,
7843 artifacts: artifacts.clone(),
7844 stem: format!("fix-{round}"),
7845 handover: None,
7846 })
7847 .await;
7848 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7851 return Ok(FixStep::Resource);
7852 }
7853 let agent_id = seat.agent.clone();
7854
7855 let mut fix = FixRecord {
7856 agent: agent_id,
7857 addressed: Vec::new(),
7858 rejected: Vec::new(),
7859 notes: String::new(),
7860 committed: false,
7861 failed: None,
7862 duration_ms: 0,
7863 continuation: None,
7864 };
7865 let mut continuation = ContinuationRecord::not_needed();
7866 let mut final_seat = seat.clone();
7867 match out {
7868 AgentOutcome::Ok(o) => {
7869 fix.duration_ms = o.duration_ms;
7870 let parsed = verdict::extract_json::<FixReport>(&o.text);
7871 let incomplete_reason = match &parsed {
7878 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7879 "the reply parsed, but it reported a command whose own CLI never \
7880 confirmed an exit status"
7881 .to_owned(),
7882 ),
7883 Ok(_) => None,
7884 Err(e) => Some(e.to_string()),
7885 };
7886 match incomplete_reason {
7887 None => {
7888 let report = parsed.expect("checked Ok above");
7889 fix.addressed = report.addressed;
7890 fix.rejected = report.rejected;
7891 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7892 }
7893 Some(reason) => {
7894 let (resumed_seat, resolved, failure, cont) = self
7895 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7896 .await;
7897 fix.duration_ms += cont.cumulative_wait_ms;
7898 continuation = cont;
7899 final_seat = resumed_seat;
7900 match resolved {
7901 Some(report) => {
7902 fix.addressed = report.addressed;
7903 fix.rejected = report.rejected;
7904 fix.notes =
7905 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7906 }
7907 None => fix.failed = failure,
7908 }
7909 }
7910 }
7911 }
7912 AgentOutcome::Dropped(o) => {
7914 fix.duration_ms = o.duration_ms;
7915 let why = o
7916 .dropped
7917 .as_ref()
7918 .map(|d| d.why.as_str())
7919 .unwrap_or("the CLI ended the stream without delivering its answer");
7920 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7921 }
7922 AgentOutcome::Quota(o) => {
7923 self.state.quota.push(QuotaLoss::from_report(
7924 final_seat.key.clone(),
7925 "fix",
7926 o.quota.as_ref(),
7927 ));
7928 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7929 }
7930 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7931 }
7932 if self.state.resource_block.is_some() {
7933 self.state.seats.insert(final_seat.key.clone(), final_seat);
7934 return Ok(FixStep::Resource);
7935 }
7936 fix.continuation = Some(continuation);
7937 self.state.seats.insert(final_seat.key.clone(), final_seat);
7938 let coordination = self.coordinate(&winner.worktree).await?;
7939 if let Ok(r) = git::rescue_commit(
7940 &winner.worktree,
7941 &format!("magi: review round {round} fixes (uncommitted work)"),
7942 )
7943 .await
7944 {
7945 self.state.note_withheld("fix", &r.withheld);
7946 }
7947 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7948 fix.committed = after != before;
7949 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7957 drop(coordination);
7958 let progressed = diff_after != patch;
7959 let commit_note = if fix.committed {
7960 "committed"
7961 } else {
7962 "NO new commit"
7963 };
7964 let tree_note = if progressed {
7965 "changed vs base"
7966 } else {
7967 "unchanged vs base"
7968 };
7969 self.state.event(
7970 "fix",
7971 match &fix.failed {
7972 Some(reason) => {
7978 format!(
7979 "round {round}: fixer's adoption report was lost ({reason}); \
7980 {commit_note}, tree {tree_note}"
7981 )
7982 }
7983 None => format!(
7984 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7985 {tree_note}{}",
7986 fix.addressed.len(),
7987 fix.rejected.len(),
7988 if continuation.outcome == ContinuationOutcome::Resumed {
7989 format!(
7990 " (adoption report recovered after {} continuation(s))",
7991 continuation.attempts
7992 )
7993 } else {
7994 String::new()
7995 },
7996 ),
7997 },
7998 );
7999 round_record.fix = Some(fix);
8000 round_record.progressed = progressed;
8001 Ok(FixStep::Recorded {
8002 report_lost: matches!(
8003 continuation.outcome,
8004 ContinuationOutcome::Exhausted
8005 | ContinuationOutcome::QuotaLost
8006 | ContinuationOutcome::NoSession
8007 ),
8008 })
8009 }
8010
8011 fn record_contested_handoff(&mut self) {
8045 if self.state.contested_handoff.is_some() {
8046 return;
8047 }
8048 let Some(contested) = self
8049 .state
8050 .reviews
8051 .last()
8052 .and_then(ReviewRound::contested_handoff)
8053 else {
8054 return;
8055 };
8056 self.state.event(
8057 "review",
8058 format!(
8059 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8060 wait for the owner's approval",
8061 contested.findings.len(),
8062 contested.rejecters.len()
8063 ),
8064 );
8065 self.state.contested_handoff = Some(contested);
8066 }
8067
8068 async fn stop_unresolved_review(
8069 &mut self,
8070 why: &str,
8071 shell: &[String],
8072 worktree: &Path,
8073 ) -> Result<()> {
8074 self.stop_reviewing_inner(why, shell, worktree, true).await
8075 }
8076
8077 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8078 self.stop_reviewing_inner(why, shell, worktree, false).await
8079 }
8080
8081 async fn stop_reviewing_inner(
8082 &mut self,
8083 why: &str,
8084 shell: &[String],
8085 worktree: &Path,
8086 repairable: bool,
8087 ) -> Result<()> {
8088 let round_idx = self.state.reviews.len() - 1;
8089 let needs_catchup_run = matches!(
8097 self.state.reviews[round_idx].e2e_status(),
8098 E2eStatus::Deferred | E2eStatus::ResourceBlocked
8099 );
8100 if needs_catchup_run {
8101 let round = self.state.reviews[round_idx].round;
8102 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8103 let commands = self.state.config.verify.e2e.clone();
8104 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8105 let cache_dir = self.state.config.cache_dir();
8106 let context = format!(
8107 "round {round}: verification unresolved, catching up before the final decision"
8108 );
8109 let wait = self.repository_wait();
8110 let (outcomes, verify_retried) = with_cache_lease(
8111 &mut self.state,
8112 cache_dir.as_deref(),
8113 "e2e",
8114 "e2e",
8115 worktree,
8116 &attempted_head,
8117 timeout,
8118 &context,
8119 &wait,
8120 |state, budget| {
8121 let shell = shell.to_vec();
8122 let commands = commands.clone();
8123 let context = context.clone();
8124 async move {
8125 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8126 .await
8127 }
8128 },
8129 )
8130 .await?;
8131 if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8132 self.state.record_base_progress(&attempted_head);
8133 }
8134 let last = &mut self.state.reviews[round_idx];
8135 last.e2e = outcomes;
8136 last.verify_retried = verify_retried;
8137 last.verified_head = Some(attempted_head);
8144 last.verified_at = Some(Timestamp::now());
8145 if verify_inconclusive(&last.e2e) {
8146 self.state.save()?;
8153 return Ok(());
8154 }
8155 last.e2e_deferred = false;
8156 if last.answered == last.expected {
8161 last.clean = round_is_clean(
8162 last.blocking,
8163 last.e2e.iter().all(CommandOutcome::ok),
8164 last.answered,
8165 last.expected,
8166 0,
8167 self.state.config.graph.incomplete_review,
8168 );
8169 }
8170 }
8171 let last = &self.state.reviews[round_idx];
8172 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8173
8174 match last.e2e_status() {
8175 E2eStatus::Failed => {
8176 let red: Vec<String> = last
8177 .e2e
8178 .iter()
8179 .filter(|o| !o.ok())
8180 .map(|o| {
8181 format!(
8182 "`{}` -> {:?}\n{}",
8183 o.command,
8184 o.code,
8185 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8186 )
8187 })
8188 .collect();
8189 let can_repair = last.blocking == 0 && last.fix.is_none();
8190 self.state.reviews[round_idx].clean = false;
8191 self.state
8192 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8193 self.state.status = RunStatus::Blocked;
8194 if repairable
8195 && can_repair
8196 && self.state.reviews.len() < self.state.config.graph.review_rounds
8197 {
8198 self.state.status = RunStatus::Reviewing;
8199 }
8200 }
8201 E2eStatus::ResourceBlocked => {
8206 self.state.event(
8207 "review",
8208 format!(
8209 "{why}; e2e could not run (resource unavailable); not \
8210 deciding yet"
8211 ),
8212 );
8213 }
8214 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8215 self.state.event(
8216 "review",
8217 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8218 );
8219 self.record_contested_handoff();
8220 self.state.status = RunStatus::Gating;
8221 }
8222 }
8223 self.state.save()?;
8224 Ok(())
8225 }
8226
8227 async fn gate(&mut self) -> Result<()> {
8230 if self.hold_base_sync().await? {
8231 return Ok(());
8232 }
8233 if self.state.status == RunStatus::Failed
8245 || self
8246 .state
8247 .base_sync
8248 .as_ref()
8249 .is_some_and(|s| s.conflict.is_some())
8250 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8251 != Some(RunStatus::Gating)
8252 {
8253 return Ok(());
8254 }
8255 let Some(winner) = self.state.winner().cloned() else {
8256 return Ok(());
8257 };
8258 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8259 self.state.reconcile_verification_head(&head);
8260 self.state.save()?;
8261 let wait = self.repository_wait();
8262 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8263 return Ok(());
8264 }
8265 if self.state.gate_ran {
8266 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8277 self.state.status = RunStatus::Blocked;
8278 self.state.save()?;
8279 }
8280 return Ok(());
8281 }
8282 self.state.status = RunStatus::Gating;
8283 let mut outcomes = self.run_gate(&winner).await?;
8284 let mut attempted_head = self.state.gate_head.clone();
8285 loop {
8286 if verify_inconclusive(&outcomes) {
8297 self.state.save()?;
8298 return Ok(());
8299 }
8300 if outcomes.iter().all(CommandOutcome::ok) {
8301 break;
8302 }
8303 let fixes_before = self.state.gate_fixes.len();
8304 match self.gate_fix_round(&winner, &outcomes).await? {
8305 GateFix::Retry => {
8306 outcomes = self.run_gate(&winner).await?;
8307 attempted_head = self.state.gate_head.clone();
8308 }
8309 GateFix::Stop => {
8310 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8311 if self.state.gate_fixes.len() > fixes_before
8312 && attempted_head.as_deref() != Some(current.as_str())
8313 && verify_current_head(&mut self.state, &winner, &wait).await?
8314 {
8315 outcomes = self.run_gate(&winner).await?;
8318 attempted_head = self.state.gate_head.clone();
8319 } else {
8320 break;
8321 }
8322 }
8323 GateFix::Defer => {
8324 self.state.save()?;
8325 return Ok(());
8326 }
8327 }
8328 }
8329 let passed = outcomes.iter().all(CommandOutcome::ok);
8330 self.state.gate = outcomes;
8331 self.state.gate_ran = true;
8332 self.state.gate_head = attempted_head;
8333 if !self.state.gate.is_empty()
8334 && !verify_inconclusive(&self.state.gate)
8335 && let Some(head) = self.state.gate_head.clone()
8336 {
8337 self.state.record_base_progress(&head);
8338 }
8339 self.state.save()?;
8340 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8341 return Ok(());
8342 }
8343 if !passed {
8344 self.state.status = RunStatus::Blocked;
8345 let spent = self.state.gate_fixes.len();
8346 self.state.event(
8347 "gate",
8348 if spent == 0 {
8349 "gate failed; not merging".to_owned()
8350 } else {
8351 format!("gate failed after {spent} gate-fix round(s); not merging")
8352 },
8353 );
8354 }
8355 self.state.save()?;
8356 Ok(())
8357 }
8358
8359 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8369 let commands = state.config.verify.pre_gate.clone();
8370 if commands.is_empty() {
8371 return;
8372 }
8373 let shell = state.config.shell();
8374 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8375 let (outcomes, _) = run_commands(
8376 state,
8377 "pre_gate",
8378 "pre_gate",
8379 0,
8380 &shell,
8381 &commands,
8382 &winner.worktree,
8383 timeout,
8384 )
8385 .await;
8386 for o in &outcomes {
8387 if !o.ok() {
8388 tracing::warn!(
8389 "pre_gate `{}` failed ({:?}); the gate decides",
8390 o.command,
8391 o.code
8392 );
8393 }
8394 state.event(
8395 "pre_gate",
8396 format!(
8397 "`{}` -> {}",
8398 o.command,
8399 if o.ok() {
8400 "pass".to_owned()
8401 } else {
8402 format!(
8403 "FAIL ({:?})\n{}",
8404 o.code,
8405 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8406 )
8407 }
8408 ),
8409 );
8410 }
8411 state.pre_gate = outcomes;
8412 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8413 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8414 Ok(head) => {
8415 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8416 state.pre_gate_commit = Some(head);
8417 }
8418 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8419 },
8420 Ok(false) => {}
8421 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8422 }
8423 if let Err(e) = state.save() {
8424 tracing::warn!("could not persist the pre_gate record: {e:#}");
8425 }
8426 }
8427
8428 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8431 let wait = self.repository_wait();
8432 run_landing_gate(&mut self.state, winner, &wait).await
8433 }
8434
8435 async fn gate_fix_round(
8448 &mut self,
8449 winner: &Candidate,
8450 outcomes: &[CommandOutcome],
8451 ) -> Result<GateFix> {
8452 let cap = if self.state.repair_grants.last().is_some_and(|g| {
8453 g.phase == crate::repair_budget::Phase::Fixing
8454 && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8455 }) {
8456 self.state.gate_fixes.len().saturating_add(1)
8457 } else {
8458 self.state.config.graph.gate_fix_rounds
8459 };
8460 let spent = self.state.gate_fixes.len();
8461 if spent >= cap {
8462 if cap > 0 {
8463 self.state.event(
8464 "gate",
8465 format!("{spent} gate-fix round(s) spent and the gate still fails"),
8466 );
8467 }
8468 return Ok(GateFix::Stop);
8469 }
8470 if !gate_fixable(outcomes) {
8471 self.state.event(
8472 "gate",
8473 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8474 command or similar); not spending a fix round on it",
8475 );
8476 return Ok(GateFix::Stop);
8477 }
8478 let min_free = self.state.config.disk.min_free_bytes;
8479 if min_free > 0 {
8480 match crate::disk::free_bytes(&winner.worktree) {
8481 Ok(free) if crate::disk::enough_space(free, min_free) => {}
8482 Ok(free) => {
8483 self.state.event(
8484 "gate",
8485 format!(
8486 "only {free} bytes free ({min_free} required by `[disk] \
8487 min_free_bytes`); not spending a fix round on a failure the disk \
8488 may explain"
8489 ),
8490 );
8491 return Ok(GateFix::Stop);
8492 }
8493 Err(e) => {
8494 self.state.event(
8495 "gate",
8496 format!("free disk space could not be measured ({e:#}); no fix round"),
8497 );
8498 return Ok(GateFix::Stop);
8499 }
8500 }
8501 }
8502
8503 let attempt = spent + 1;
8504 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8505 let base = self.landing_base();
8506 let fix_prompt = prompt::gate_fix_in(
8507 &self.state.instruction,
8508 &failed,
8509 attempt,
8510 cap,
8511 &self.state.config.graph.language,
8512 &self.state.config.graph.github_language,
8513 );
8514 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8515 let sessions = self.state.config.graph.sessions;
8516 let artifacts = self.merge_repair_artifacts();
8517 self.state.event(
8518 "gate",
8519 format!("gate failed; gate-fix round {attempt} of {cap}"),
8520 );
8521 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8522 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8523 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8526 head: before.clone(),
8527 passed: false,
8528 outcomes: Vec::new(),
8529 });
8530 self.state.save()?;
8531 let (_, seat, out) = self
8532 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8533 prompt: fix_prompt.clone(),
8534 spec,
8535 seat,
8536 cwd: winner.worktree.clone(),
8537 timeout,
8538 allow_write: true,
8539 sessions,
8540 artifacts: artifacts.clone(),
8541 stem: format!("gate-fix-{attempt}"),
8542 handover: None,
8543 })
8544 .await;
8545 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8548 return Ok(GateFix::Defer);
8549 }
8550 let mut record = GateFixRecord {
8551 agent: seat.agent.clone(),
8552 failed,
8553 notes: String::new(),
8554 committed: false,
8555 error: None,
8556 };
8557 match out {
8558 AgentOutcome::Ok(o) => {
8559 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8562 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8563 }
8564 }
8565 AgentOutcome::Dropped(_) => {
8566 record.error = Some("the CLI dropped the stream".to_owned());
8567 }
8568 AgentOutcome::Quota(o) => {
8569 self.state.quota.push(QuotaLoss::from_report(
8570 seat.key.clone(),
8571 "gate-fix",
8572 o.quota.as_ref(),
8573 ));
8574 record.error = Some("rate limited (quota); fixer could not run".to_owned());
8575 }
8576 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8577 }
8578 self.state.seats.insert(seat.key.clone(), seat);
8579 let coordination = self.coordinate(&winner.worktree).await?;
8580 if let Ok(r) = git::rescue_commit(
8581 &winner.worktree,
8582 &format!("magi: gate fix {attempt} (uncommitted work)"),
8583 )
8584 .await
8585 {
8586 self.state.note_withheld("gate-fix", &r.withheld);
8587 }
8588 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8589 record.committed = after != before;
8590 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8591 let note = record.error.clone();
8592 self.state.gate_fixes.push(record);
8593 self.state.save()?;
8594 if !changed {
8595 self.state.event(
8596 "gate",
8597 match note {
8598 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8599 None => format!("gate-fix round {attempt}: the tree did not change"),
8600 },
8601 );
8602 return Ok(GateFix::Stop);
8603 }
8604 self.state.event(
8605 "gate",
8606 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8607 );
8608
8609 drop(coordination);
8610 let wait = self.repository_wait();
8611 check_gate_repair_e2e(&mut self.state, winner, &wait).await
8612 }
8613
8614 async fn guarded_pr_message(
8621 &mut self,
8622 winner: &Candidate,
8623 facts: Option<&BranchFacts>,
8624 posting: bool,
8625 ) -> Result<Option<PrMessage>> {
8626 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8627 let head = match final_head.as_deref() {
8628 Some(h) => crate::pr_outcome::FinalHead::At(h),
8629 None => crate::pr_outcome::FinalHead::Unknown,
8630 };
8631 let summarized = if posting {
8633 Some(self.summarize_change(winner, final_head.as_deref()).await)
8634 } else {
8635 None
8636 };
8637 if self.state.resource_block.is_some() {
8642 self.state.event(
8643 "pr-summary",
8644 "a shared resource was busy; nothing was published, resume to summarize again",
8645 );
8646 return Ok(None);
8647 }
8648 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8649 anyhow::bail!(
8650 "the final commit moved during public description generation; retry before publishing"
8651 );
8652 }
8653 let note = match (&summarized, &self.state.change_summary) {
8654 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8655 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8656 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8657 (None, _) => ChangeNote::NotAsked,
8658 };
8659 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8660 if !posting {
8661 let identity = crate::scrub::Identity::current();
8662 return Ok(Some(PrMessage {
8663 title: crate::scrub::scrub(&pr.title, &identity),
8664 body: crate::scrub::scrub(&pr.body, &identity),
8665 }));
8666 }
8667 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8670 if let Some(known) = self
8671 .state
8672 .github_text
8673 .clone()
8674 .filter(|g| g.fingerprint == raw_fp)
8675 {
8676 return self.settle_withheld(winner, &pr, known).await;
8677 }
8678 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8679 let mut violations =
8680 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8681 let identity = crate::scrub::Identity::current();
8682 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8683 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8684 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8685 {
8686 violations.push(crate::github_text::Violation::SensitiveData);
8687 }
8688 let has_language = violations
8692 .iter()
8693 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8694 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8695 self.state.event(
8696 "github-text",
8697 if has_language {
8698 format!("description rejected: {violations:?}; requesting one rewrite")
8699 } else {
8700 format!("description rejected: {violations:?}; asking the owner")
8701 },
8702 );
8703 let mut rewritten = false;
8704 let mut rejected_rewrite = None;
8705 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8706 {
8707 let output = text;
8708 if let Some(title) = summary_title(&output) {
8709 let raw_body = summary_without_title(&output);
8712 let clean_title = crate::scrub::scrub(&title, &identity);
8713 let mut body = crate::scrub::scrub(&raw_body, &identity);
8714 let redacted = clean_title != title || body != raw_body;
8715 let title = clean_title;
8716 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8717 if !complete {
8718 self.state.event(
8719 "github-text",
8720 "description rewrite omitted required findings",
8721 );
8722 }
8723 if !body.trim().is_empty() {
8724 body.push_str("\n\n");
8725 body.push_str(&public_pr_evidence(&self.state, head));
8726 }
8727 let marker = format!("magi:run/{}", self.state.id);
8728 if !body.trim().is_empty() && !body.contains(&marker) {
8729 body.push_str(&format!("\n\n{marker}"));
8730 }
8731 let rewrite = PrMessage {
8732 title: title.clone(),
8733 body: body.clone(),
8734 };
8735 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8736 let mut failed = crate::github_text::check_for(
8737 &self.state.config.graph,
8738 &title,
8739 &body,
8740 redecision,
8741 );
8742 if (!crate::github_text::shareable(&title)
8743 || !crate::github_text::shareable(&body))
8744 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8745 {
8746 failed.push(crate::github_text::Violation::SensitiveData);
8747 }
8748 if complete
8749 && !title.trim().is_empty()
8750 && !body.trim().is_empty()
8751 && !failed.is_empty()
8752 {
8753 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8754 title: title.clone(),
8755 body: body.clone(),
8756 failed: failed
8757 .iter()
8758 .map(|v| crate::github_text::category(*v).to_owned())
8759 .collect(),
8760 });
8761 }
8762 if complete
8763 && !title.trim().is_empty()
8764 && !raw_body.trim().is_empty()
8765 && failed.is_empty()
8766 {
8767 decision = redecision;
8768 pr = PrMessage { title, body };
8769 rewritten = true;
8770 self.state
8771 .event("github-text", "description rewrite passed");
8772 if redacted {
8773 self.state.event(
8774 "github-text",
8775 "sensitive data removed from description rewrite",
8776 );
8777 }
8778 }
8779 }
8780 }
8781 if !rewritten {
8782 self.state.event(
8783 "github-text",
8784 "description rewrite unavailable or rejected; asking the owner",
8785 );
8786 if let Some(w) =
8787 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8788 {
8789 self.retire_github_text_question("the withheld text changed");
8792 let record = crate::run::GithubTextAsk {
8793 rewrite: rejected_rewrite,
8794 fingerprint: raw_fp,
8795 title: w.title,
8796 body: w.body,
8797 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8798 question: None,
8799 asks: 0,
8800 resolved: false,
8801 chosen_title: None,
8802 choice: None,
8803 publication: None,
8804 };
8805 return self.settle_withheld(winner, &pr, record).await;
8806 }
8807 }
8808 }
8809 let (title, body) =
8810 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8811 Ok(Some(PrMessage { title, body }))
8812 }
8813
8814 async fn summarize_change(
8825 &mut self,
8826 winner: &Candidate,
8827 head: Option<&str>,
8828 ) -> std::result::Result<(), String> {
8829 let Some(head) = head else {
8830 return Err("the final commit could not be read".to_owned());
8831 };
8832 let remote = self.state.config.merge.remote.clone();
8833 let tracking = format!("{remote}/{}", self.state.base_branch);
8834 if !git::rev_exists(&winner.worktree, &tracking).await {
8835 return Err(format!("the remote base `{tracking}` could not be read"));
8836 }
8837 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8838 Ok(b) if !b.is_empty() => b,
8839 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8840 };
8841 let evidence = crate::pr_outcome::summary_evidence(
8842 &self.state,
8843 crate::pr_outcome::FinalHead::At(head),
8844 );
8845 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8846 if self
8847 .state
8848 .change_summary
8849 .as_ref()
8850 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8851 {
8852 return Ok(());
8853 }
8854 if self.state.change_summary.take().is_some() {
8855 self.state.event(
8856 "pr-summary",
8857 "the final diff or evidence changed; the earlier change summary was discarded",
8858 );
8859 }
8860 if let Err(e) = write_artifact(
8862 &self.state,
8863 &format!("public-description-{fingerprint}.md"),
8864 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8865 ) {
8866 tracing::warn!("could not save public description evidence: {e:#}");
8867 }
8868 let chain = match agent::pick_chain(
8869 &self.state.config.agents,
8870 self.state.config.roles.synthesizer.as_ref(),
8871 &agent::installed,
8872 "synthesizer",
8873 ) {
8874 Ok(chain) => chain,
8875 Err(e) => {
8876 tracing::warn!("no agent for the change summary: {e:#}");
8877 self.state.event("pr-summary", "no summary agent available");
8878 return Err("no agent was available to summarize it".to_owned());
8879 }
8880 };
8881 let subjects = git::subjects(&winner.worktree, &base, head)
8882 .await
8883 .unwrap_or_default();
8884 let stat = git::diff_stat(&winner.worktree, &base, head)
8885 .await
8886 .unwrap_or_default();
8887 let stat: String = stat.chars().take(6_000).collect();
8888 let mut prompt = crate::prompt::change_summary_in(
8889 &base,
8890 head,
8891 &subjects,
8892 stat.trim(),
8893 &self.state.config.graph.github_language,
8894 );
8895 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8896 let sessions = self.state.config.graph.sessions;
8897 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8898 let prompts = self.state.config.prompts.clone();
8899 let cache = self.state.config.cache_dir();
8900 let run = self.state.id.clone();
8901 for (i, spec) in chain.into_iter().enumerate() {
8902 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8903 let github_spec = match agent::for_github_text(
8904 &spec,
8905 &self.state.config.graph.github_language,
8906 &winner.worktree,
8907 ) {
8908 Ok(spec) => spec,
8909 Err(e) => {
8910 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8911 continue;
8912 }
8913 };
8914 let job = SeatJob {
8915 spec: github_spec,
8916 seat,
8917 cwd: winner.worktree.clone(),
8918 prompt: prompt.clone(),
8919 timeout,
8920 allow_write: false,
8921 sessions,
8922 artifacts: agent::artifacts_dir(&self.state.dir()),
8923 stem: if i == 0 {
8924 "change-summary".to_owned()
8925 } else {
8926 format!("change-summary-{}", spec.id)
8927 },
8928 handover: None,
8929 };
8930 let ctx = WaveCtx {
8931 run: &run,
8932 node: "pr-summary",
8933 prompts: &prompts,
8934 cache: cache.as_deref(),
8935 round: None,
8936 carry_seats: false,
8937 pauses: Some((&self.pause, &self.interrupt)),
8938 };
8939 let (seat, outcome) =
8940 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8941 self.state.seats.insert(seat.key.clone(), seat);
8942 let text = match outcome {
8943 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8944 AgentOutcome::ResourceBlocked(why) => {
8945 if self.state.resource_block.is_none() {
8948 self.state.resource_block = Some(crate::run::ResourceBlock {
8949 cause: crate::run::ResourceBlockCause::Other,
8950 node: "pr-summary".to_owned(),
8951 kind: "shared resource".to_owned(),
8952 detail: why,
8953 at: Timestamp::now(),
8954 });
8955 }
8956 return Err("a shared resource was busy".to_owned());
8957 }
8958 _ => continue,
8959 };
8960 let blind = &self.state.config.blind;
8961 let Some(title) = summary_title(&text) else {
8962 continue;
8963 };
8964 let body: String = summary_without_title(&text)
8965 .lines()
8966 .filter(|l| !l.contains("magi:run/"))
8967 .collect::<Vec<_>>()
8968 .join("\n");
8969 let body = blind::sanitize_prose(body.trim(), blind);
8970 let title = blind::sanitize_prose(&title, blind);
8971 if body.trim().is_empty() || title.trim().is_empty() {
8972 continue;
8973 }
8974 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8975 continue;
8976 }
8977 if git::rev_parse(&winner.worktree, "HEAD")
8979 .await
8980 .ok()
8981 .as_deref()
8982 != Some(head)
8983 {
8984 return Err("the branch moved while the summary was written".to_owned());
8985 }
8986 if git::merge_base(&winner.worktree, &tracking, head)
8987 .await
8988 .ok()
8989 .as_deref()
8990 != Some(base.as_str())
8991 || crate::pr_outcome::summary_evidence(
8992 &self.state,
8993 crate::pr_outcome::FinalHead::At(head),
8994 ) != evidence
8995 {
8996 return Err("the summary evidence changed while it was written".to_owned());
8997 }
8998 self.state.change_summary = Some(crate::run::ChangeSummary {
8999 head: head.to_owned(),
9000 base,
9001 evidence: fingerprint,
9002 title,
9003 body,
9004 });
9005 self.state.event(
9006 "pr-summary",
9007 format!("change summary written by {} for the final diff", spec.id),
9008 );
9009 if let Err(e) = self.state.save() {
9010 tracing::warn!("could not save the change summary: {e:#}");
9011 }
9012 return Ok(());
9013 }
9014 self.state
9015 .event("pr-summary", "no agent produced a change summary");
9016 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9017 }
9018
9019 async fn rewrite_pr_text(
9029 &mut self,
9030 winner: &Candidate,
9031 pr: &PrMessage,
9032 violations: &[crate::github_text::Violation],
9033 ) -> Option<String> {
9034 let prompt = format!(
9035 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9036 pr.title, pr.body
9037 );
9038 let sessions = self.state.config.graph.sessions;
9039 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9040 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9043 let key = format!("impl-{}", winner.label);
9044 if let Some(spec) = self
9045 .state
9046 .config
9047 .agents
9048 .iter()
9049 .find(|a| a.id == winner.agent)
9050 .cloned()
9051 && let Some(existing) = self.state.seats.get(&key)
9052 && existing.agent == spec.id
9053 && agent::has_session(spec.kind, existing, sessions)
9054 {
9055 attempts.push((spec, Some(existing.clone())));
9056 }
9057 match agent::pick_chain(
9058 &self.state.config.agents,
9059 self.state.config.roles.language_judge.as_ref(),
9060 &agent::installed,
9061 "language_judge",
9062 ) {
9063 Ok(chain) => {
9064 for spec in chain {
9065 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9066 continue;
9067 }
9068 attempts.push((spec, None));
9069 }
9070 }
9071 Err(e) if attempts.is_empty() => {
9072 tracing::warn!("no agent for the description rewrite: {e:#}");
9073 self.state
9074 .event("github-text", "no rewrite agent available");
9075 return None;
9076 }
9077 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9078 }
9079 let prompts = self.state.config.prompts.clone();
9080 let cache = self.state.config.cache_dir();
9081 let run = self.state.id.clone();
9082 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9083 let resumed = resumed_seat.is_some();
9084 let seat = resumed_seat.unwrap_or_else(|| {
9085 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9086 });
9087 let github_spec = match agent::for_github_text(
9088 &spec,
9089 &self.state.config.graph.github_language,
9090 &winner.worktree,
9091 ) {
9092 Ok(spec) => spec,
9093 Err(e) => {
9094 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9095 continue;
9096 }
9097 };
9098 let job = SeatJob {
9099 spec: github_spec,
9100 seat,
9101 cwd: winner.worktree.clone(),
9102 prompt: prompt.clone(),
9103 timeout: retry_budget(timeout_full, resumed),
9104 allow_write: false,
9105 sessions,
9106 artifacts: agent::artifacts_dir(&self.state.dir()),
9107 stem: if i == 0 {
9108 "github-text-rewrite".to_owned()
9109 } else {
9110 format!("github-text-rewrite-{}", spec.id)
9111 },
9112 handover: None,
9113 };
9114 let ctx = WaveCtx {
9115 run: &run,
9116 node: "github-text",
9117 prompts: &prompts,
9118 cache: cache.as_deref(),
9119 round: None,
9120 carry_seats: false,
9121 pauses: Some((&self.pause, &self.interrupt)),
9122 };
9123 let (seat, outcome) =
9124 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9125 self.state.seats.insert(seat.key.clone(), seat);
9126 match outcome {
9127 AgentOutcome::Ok(o)
9128 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9129 {
9130 return Some(o.text);
9131 }
9132 AgentOutcome::ResourceBlocked(_) => return None,
9133 _ => {}
9134 }
9135 }
9136 None
9137 }
9138
9139 fn retire_github_text_question(&mut self, why: &str) {
9141 let Some(id) = self
9142 .state
9143 .github_text
9144 .as_ref()
9145 .and_then(|g| g.question.clone())
9146 else {
9147 return;
9148 };
9149 let _ = ask::Questions::open().update(&id, |q| {
9150 q.abandon(why);
9151 Ok(())
9152 });
9153 }
9154
9155 async fn settle_withheld(
9159 &mut self,
9160 winner: &Candidate,
9161 pr: &PrMessage,
9162 mut g: crate::run::GithubTextAsk,
9163 ) -> Result<Option<PrMessage>> {
9164 use crate::github_text as gt;
9165 let store = ask::Questions::open();
9166 let timeout = self.state.config.graph.answer_timeout;
9167 if !g.resolved {
9168 let mut standing = g
9169 .question
9170 .as_ref()
9171 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9172 if let Some(q) = standing.as_ref()
9173 && q.status.open()
9174 && gt::expired(q, timeout)
9175 {
9176 let why = format!("no answer within {}s of asking", timeout.max(1));
9177 standing = store
9178 .update(&q.id, |q| {
9179 q.abandon(&why);
9180 Ok(())
9181 })
9182 .ok()
9183 .map(|(q, ())| q);
9184 }
9185 let mut ask_again: Option<Option<Vec<String>>> = None;
9186 match standing {
9187 None if g.asks <= 1 => {
9190 g.asks = 0;
9191 ask_again = Some(None);
9192 }
9193 None => {
9194 g.resolved = true;
9195 g.chosen_title = None;
9196 g.choice = Some(GithubTextChoice::Fallback);
9197 }
9198 Some(q) if q.status.open() => {
9199 self.park_on_github_text(&g)?;
9200 return Ok(None);
9201 }
9202 Some(q) => match gt::read_reply(&q) {
9203 gt::Reply::Fallback => {
9204 g.resolved = true;
9205 g.chosen_title = None;
9206 g.choice = Some(GithubTextChoice::Fallback);
9207 }
9208 gt::Reply::Original => {
9209 g.resolved = true;
9210 g.chosen_title = None;
9211 g.choice = Some(GithubTextChoice::Original);
9212 }
9213 gt::Reply::Rewrite => {
9214 g.resolved = true;
9215 g.chosen_title = None;
9216 g.choice = Some(if g.rewrite.is_some() {
9217 GithubTextChoice::Rewrite
9218 } else {
9219 GithubTextChoice::Fallback
9220 });
9221 }
9222 gt::Reply::Title(text) => {
9223 let first = text
9224 .lines()
9225 .map(str::trim)
9226 .find(|l| !l.is_empty())
9227 .unwrap_or("");
9228 let decision = if first.is_empty() {
9229 None
9230 } else {
9231 let candidate = PrMessage {
9232 title: first.to_owned(),
9233 body: String::new(),
9234 };
9235 self.judge_pr_language(&winner.worktree, &candidate).await
9236 };
9237 match gt::vet_title(
9238 &text,
9239 decision,
9240 self.state.config.graph.github_english(),
9241 ) {
9242 Ok(title) => {
9243 g.resolved = true;
9244 g.chosen_title = Some(title);
9245 g.choice = Some(GithubTextChoice::MyTitle);
9246 }
9247 Err(cats) if g.asks < 2 => {
9248 ask_again = Some(Some(
9249 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9250 ))
9251 }
9252 Err(_) => {
9253 g.resolved = true;
9254 g.chosen_title = None;
9255 g.choice = Some(GithubTextChoice::Fallback);
9256 }
9257 }
9258 }
9259 },
9260 }
9261 if let Some(rejected) = ask_again {
9262 let w = gt::Withheld {
9265 title: g.title,
9266 body: g.body,
9267 categories: g.categories.clone(),
9268 };
9269 let artifact = match crate::run::write_artifact(
9272 &self.state,
9273 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
9274 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9275 ) {
9276 Ok(p) => Some(p),
9277 Err(e) => {
9278 tracing::warn!("could not write the withheld-text artifact: {e:#}");
9279 None
9280 }
9281 };
9282 let (detail, comparison) = gt::question_content(
9283 &w,
9284 &pr.title,
9285 &pr.body,
9286 &self.state.id,
9287 rejected.as_deref(),
9288 artifact.as_deref(),
9289 g.rewrite.as_ref(),
9290 );
9291 let mut q = ask::Question::new(
9292 self.state.id.clone(),
9293 gt::ASK_NODE.to_owned(),
9294 gt::ASK_SEAT.to_owned(),
9295 gt::question_summary(&self.state.config.graph.language, &w),
9296 detail,
9297 gt::question_choices(&w, g.rewrite.as_ref()),
9298 );
9299 q.posting_comparison = Some(comparison);
9300 q.answer_timeout = timeout;
9301 g.question = Some(q.id.clone());
9304 g.asks += 1;
9305 self.state.github_text = Some(g.clone());
9306 self.state.event(
9307 "github-text",
9308 format!("asking the owner about the withheld text ({})", q.short()),
9309 );
9310 self.state.save()?;
9311 store
9312 .put_executable(&mut q)
9313 .context("file the posting-gate question")?;
9314 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9315 tracing::warn!(
9316 "could not notify about posting-gate question {}: {e:#}",
9317 q.short()
9318 );
9319 }
9320 self.park_on_github_text(&g)?;
9321 return Ok(None);
9322 }
9323 }
9324 let resolved = g.chosen_title.clone();
9325 let original = g.choice == Some(GithubTextChoice::Original);
9326 self.state.event(
9328 "github-text",
9329 if original {
9330 format!(
9331 "posting the original text for: {}",
9332 match (g.title, g.body) {
9333 (true, true) => "title, description",
9334 (true, false) => "title",
9335 _ => "description",
9336 }
9337 )
9338 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9339 "posting the saved rewrite title and description".to_owned()
9340 } else if resolved.is_some() {
9341 "posting the owner's replacement title".to_owned()
9342 } else {
9343 "posting the neutral text for the withheld fields".to_owned()
9344 },
9345 );
9346 self.state.github_text = Some(g.clone());
9347 self.state.save()?;
9348 if g.choice == Some(GithubTextChoice::Rewrite)
9349 && let Some(r) = g.rewrite.as_ref()
9350 {
9351 return Ok(Some(PrMessage {
9352 title: r.title.clone(),
9353 body: r.body.clone(),
9354 }));
9355 }
9356 let w = gt::Withheld {
9357 title: g.title,
9358 body: g.body,
9359 categories: g.categories.clone(),
9360 };
9361 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9362 if let Some(t) = resolved {
9363 title = t;
9364 }
9365 let english = gt::LanguageDecision {
9367 title_english: true,
9368 body_english: true,
9369 body_complete: true,
9370 };
9371 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9372 if original {
9373 if g.title {
9376 title = pr.title.clone();
9377 }
9378 if g.body {
9379 body = pr.body.clone();
9380 }
9381 }
9382 Ok(Some(PrMessage { title, body }))
9383 }
9384
9385 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9386 self.state.github_text = Some(g.clone());
9387 self.state.end_pass_parked(ParkCause::Question);
9388 self.state.parked = true;
9389 self.state.event(
9390 "github-text",
9391 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9392 );
9393 self.state.save()
9394 }
9395
9396 async fn judge_pr_language(
9399 &mut self,
9400 cwd: &Path,
9401 pr: &PrMessage,
9402 ) -> Option<crate::github_text::LanguageDecision> {
9403 self.state.config.roles.language_judge.as_ref()?;
9404 if !self.state.config.graph.language_gate() {
9405 return None;
9406 }
9407 let decision =
9408 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9409 self.state.event(
9410 "github-text",
9411 if decision.is_some() {
9412 "language decided by the language judge"
9413 } else {
9414 "language judge unavailable; using built-in heuristics"
9415 },
9416 );
9417 decision
9418 }
9419
9420 async fn merge(&mut self) -> Result<()> {
9421 if self.resource_halt()? || self.hold_base_sync().await? {
9422 return Ok(());
9423 }
9424 if self
9439 .state
9440 .base_sync
9441 .as_ref()
9442 .is_some_and(|s| s.conflict.is_some())
9443 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9444 != Some(RunStatus::Gating)
9445 || !self.state.gate_allows_progress()
9453 {
9454 return Ok(());
9455 }
9456 if self.state.merge.is_some() && !self.state.pr_publication_pending {
9465 return Ok(());
9466 }
9467 let Some(winner) = self.state.winner().cloned() else {
9468 return Ok(());
9469 };
9470 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9471 if self.state.gate_head.as_deref() != Some(head.as_str()) {
9472 self.state.status = RunStatus::Blocked;
9473 self.state.event(
9474 "gate",
9475 "winner changed after gate; fresh verification required",
9476 );
9477 self.state.save()?;
9478 return Ok(());
9479 }
9480 let wait = self.repository_wait();
9481 if !verify_current_head(&mut self.state, &winner, &wait).await? {
9482 return Ok(());
9483 }
9484 let repo = self.state.repo.clone();
9485 let base = self.state.base_branch.clone();
9486 let mode = self.state.config.merge.mode;
9487 let style = self.state.config.merge.style;
9488 let facts = if is_review_run(&self.state) {
9489 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9490 let start = review_base(
9491 &repo,
9492 &self.state.config.merge.remote,
9493 &base,
9494 &self.state.base_commit,
9495 &winner.branch,
9496 )
9497 .await;
9498 branch_facts(&repo, &start, &winner.branch).await
9499 } else {
9500 None
9501 };
9502 let leaked = if is_review_run(&self.state) {
9505 leaked_subjects(&self.state, &winner.branch).await
9506 } else {
9507 Some(Vec::new())
9508 };
9509 let Some(pr) = self
9513 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9514 .await?
9515 else {
9516 return Ok(());
9517 };
9518 let message = pr.commit_message();
9519
9520 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9521 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9522 return Ok(());
9523 }
9524 let outcome = match mode {
9525 MergeMode::None => MergeOutcome {
9526 mode,
9527 ok: true,
9528 detail: manual_merge_command(style, &repo, &winner.branch, &message),
9529 empty: false,
9530 },
9531 MergeMode::Pr | MergeMode::Local
9532 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9533 {
9534 MergeOutcome {
9535 mode,
9536 ok: false,
9537 detail: empty_candidate_detail(&self.state, &base),
9538 empty: true,
9539 }
9540 }
9541 MergeMode::Local => {
9542 let on = git::current_branch(&repo).await?;
9543 if on.as_deref() != Some(base.as_str()) {
9544 MergeOutcome {
9545 mode,
9546 ok: false,
9547 detail: format!(
9548 "{} has {} checked out, not the base branch {base}",
9549 repo.display(),
9550 on.unwrap_or_else(|| "a detached HEAD".to_owned())
9551 ),
9552 empty: false,
9553 }
9554 } else if !git::is_clean(&repo).await? {
9555 MergeOutcome {
9556 mode,
9557 ok: false,
9558 detail: format!("{} is dirty; refusing to merge", repo.display()),
9559 empty: false,
9560 }
9561 } else {
9562 let out = match style {
9563 MergeStyle::Merge => {
9564 git::merge_no_ff(&repo, &winner.branch, &message).await?
9565 }
9566 MergeStyle::Squash => {
9567 git::merge_squash(&repo, &winner.branch, &message).await?
9568 }
9569 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9570 };
9571 MergeOutcome {
9572 mode,
9573 ok: out.ok(),
9574 detail: if out.ok() { out.stdout } else { out.stderr },
9575 empty: false,
9576 }
9577 }
9578 }
9579 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9580 self.state.pr_publication_pending = true;
9581 self.state.save()?;
9582 let out =
9583 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9584 self.state.pr_publication_pending = out.is_err();
9585 MergeOutcome {
9586 mode,
9587 ok: out.is_ok(),
9588 detail: match out {
9589 Ok(url) => url,
9590 Err(e) => format!("{e:#}"),
9591 },
9592 empty: false,
9593 }
9594 }
9595 MergeMode::Pr => {
9596 let remote = self.state.config.merge.remote.clone();
9597 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9598 let pushed = if self.state.foreign_checkout.is_some() {
9599 let refspec = format!("{delivery_head}:refs/heads/{public}");
9600 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9601 } else {
9602 git::push(&winner.worktree, &remote, &winner.branch).await?
9603 };
9604 if !pushed.ok() {
9605 MergeOutcome {
9606 mode,
9607 ok: false,
9608 detail: pushed.stderr,
9609 empty: false,
9610 }
9611 } else {
9612 if self.state.foreign_checkout.is_some() {
9613 self.state.pr_pushed_head = Some(delivery_head.clone());
9614 }
9615 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9621 let out = match pr_merge_plan(found) {
9622 PrPlan::Create => {
9623 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9624 .await
9625 }
9626 PrPlan::Adopt { url, title } => {
9627 self.state
9628 .event("merge", format!("Pr: adopted open pull request {url}"));
9629 if self.state.explicit_github_text_selection() {
9630 land::set_pr_text_verbatim(
9633 &winner.worktree,
9634 &url,
9635 &pr.title,
9636 &pr.body,
9637 )
9638 .await
9639 .map(|()| url)
9640 } else {
9641 if title != pr.title
9642 && (!is_review_run(&self.state)
9643 || leaked
9644 .as_deref()
9645 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9646 && let Err(e) =
9647 if self.state.github_text.as_ref().is_some_and(|g| {
9648 g.resolved
9649 && g.title
9650 && g.choice == Some(GithubTextChoice::Original)
9651 }) {
9652 land::set_pr_title_verbatim(
9654 &winner.worktree,
9655 &url,
9656 &pr.title,
9657 )
9658 .await
9659 } else {
9660 land::set_pr_title(
9661 &mut self.state,
9662 &winner.worktree,
9663 &url,
9664 &pr.title,
9665 )
9666 .await
9667 }
9668 {
9669 tracing::warn!("could not refresh title of {url}: {e:#}");
9670 self.state
9671 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9672 }
9673 Ok(url)
9674 }
9675 }
9676 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9677 };
9678 match out {
9679 Ok(url) => MergeOutcome {
9680 mode,
9681 ok: true,
9682 detail: url,
9683 empty: false,
9684 },
9685 Err(e) => MergeOutcome {
9686 mode,
9687 ok: false,
9688 detail: e.to_string(),
9689 empty: false,
9690 },
9691 }
9692 }
9693 }
9694 };
9695
9696 self.state.status = match (mode, outcome.ok) {
9697 (MergeMode::None, _) => RunStatus::Ready,
9698 (_, true) if outcome.empty => RunStatus::Ready,
9699 (MergeMode::Pr, true) => RunStatus::Ready,
9700 (_, true) => RunStatus::Merged,
9701 (_, false) => RunStatus::Blocked,
9702 };
9703 self.state.event(
9704 "merge",
9705 format!(
9706 "{:?}: {}",
9707 mode,
9708 outcome.detail.lines().next().unwrap_or("")
9709 ),
9710 );
9711 if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9712 self.state.record_github_text_publication(&outcome.detail);
9713 }
9714 self.state.merge = Some(outcome);
9715 self.state.save()?;
9716
9717 if self.state.config.graph.land
9723 && mode == MergeMode::Pr
9724 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9725 {
9726 self.run_land().await?;
9727 }
9728 self.settle_questions();
9733 Ok(())
9734 }
9735
9736 async fn run_land(&mut self) -> Result<()> {
9747 crate::pr_head::ensure_owner(&self.state)?;
9748 let url = self
9749 .state
9750 .pr
9751 .as_ref()
9752 .map(|pr| pr.url.clone())
9753 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9754 .unwrap_or_default();
9755 let url = url.lines().next().unwrap_or("").trim().to_owned();
9756 if !url.starts_with("http") {
9757 return Ok(());
9758 }
9759 loop {
9762 match land::land(&mut self.state, &url).await {
9763 Ok(_)
9764 if self
9765 .state
9766 .merge_repairs
9767 .last()
9768 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9769 {
9770 if self.repair_merge_review().await? {
9771 continue;
9772 }
9773 }
9774 Ok(_) if self.state.resource_block.is_some() => {
9775 return self.stop_for_resource();
9776 }
9777 Ok(pr) if self.state.parked => {
9778 let _ = pr;
9782 }
9783 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9784 self.state.status = RunStatus::Superseded;
9785 self.state.save()?;
9786 }
9787 Ok(pr) => {
9788 self.state.status = match pr.state {
9789 land::PrLifecycle::Merged => RunStatus::Merged,
9790 _ => RunStatus::Blocked,
9791 };
9792 if bump::should_release_bump(self.state.status)
9799 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9800 {
9801 self.state
9807 .event("bump", format!("release bump skipped: {e:#}"));
9808 }
9809 if self.state.status == RunStatus::Merged {
9812 crate::followup::after_merge(&mut self.state, &pr.url).await;
9813 }
9814 self.state.save()?;
9815 }
9816 Err(e) => {
9817 if e.is::<crate::repo_lock::ResourceBlocked>() {
9818 return Err(e);
9819 }
9820 self.state.status = RunStatus::Blocked;
9821 self.state.event("land", format!("gave up: {e}"));
9822 self.state.save()?;
9823 }
9824 }
9825 break;
9826 }
9827 Ok(())
9828 }
9829
9830 async fn repair_merge_review(&mut self) -> Result<bool> {
9833 use crate::run::MergeRepairPhase as Phase;
9834 crate::pr_head::ensure_owner(&self.state)?;
9835 let i = self.state.merge_repairs.len() - 1;
9836 let request = self.state.merge_repairs[i].clone();
9837 if request.phase == Phase::Fixing {
9838 let restored = match self.state.winner().cloned() {
9839 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9840 None => Err(anyhow::anyhow!("repair has no winning branch")),
9841 };
9842 let why = match restored {
9843 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9844 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9845 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9846 };
9847 self.hold_merge_repair(i, why)?;
9848 return Ok(false);
9849 }
9850 if request.phase == Phase::Reviewing && self.state.gate_ran {
9855 let winner = self
9856 .state
9857 .winner()
9858 .context("repair has no winning branch")?;
9859 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9860 if self.state.gate_head.as_deref() != Some(head.as_str())
9861 || !git::is_clean(&winner.worktree).await?
9862 {
9863 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9864 return Ok(false);
9865 }
9866 }
9867 let failure = if i >= self.state.config.graph.land_rounds
9868 || self.state.config.graph.review_rounds == 0
9869 {
9870 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9871 } else {
9872 None
9873 };
9874 if let Some(why) = failure {
9875 self.hold_merge_repair(i, why)?;
9876 return Ok(false);
9877 }
9878 if request.phase == Phase::Requested {
9879 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9880 Ok(pr) => pr,
9881 Err(e) => {
9882 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9883 return Ok(false);
9884 }
9885 };
9886 self.state.merge_repairs[i].phase = Phase::Fixing;
9887 self.state.land_approval = None;
9888 self.state.land_armed_head = None;
9889 self.state.save()?;
9890 let evidence = serde_json::to_string_pretty(&request)?;
9893 let reason = format!(
9894 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9895 );
9896 let budget = self.state.config.graph.land_rounds;
9897 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9898 match result {
9899 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9900 Ok(land::Fixed::Deferred) => {
9901 let worktree = self
9905 .state
9906 .winner()
9907 .context("repair has no winning branch")?
9908 .worktree
9909 .clone();
9910 self.graft_restore(&worktree, &request.head).await?;
9911 self.state.merge_repairs[i].phase = Phase::Requested;
9912 self.stop_for_resource()?;
9913 return Ok(false);
9914 }
9915 Ok(land::Fixed::Failed(why)) => {
9916 self.hold_merge_repair(
9917 i,
9918 format!(
9919 "repair failed: {why}; inspect the branch and run an explicit review"
9920 ),
9921 )?;
9922 return Ok(false);
9923 }
9924 Err(e) => {
9925 self.hold_merge_repair(
9926 i,
9927 format!(
9928 "repair failed: {e:#}; inspect the branch and run an explicit review"
9929 ),
9930 )?;
9931 return Ok(false);
9932 }
9933 }
9934 self.begin_merge_repair_review(i)?;
9938 }
9939 self.review_loop().await?;
9940 self.gate().await?;
9941 if self.resource_halt()? || self.state.parked {
9942 return Ok(false);
9943 }
9944 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9945 != Some(RunStatus::Gating)
9946 || !self.state.gate_allows_progress()
9947 {
9948 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9949 return Ok(false);
9950 }
9951 let winner = self
9954 .state
9955 .winner()
9956 .cloned()
9957 .context("repair has no winning branch")?;
9958 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9959 if self.state.gate_head.as_deref() != Some(after.as_str())
9960 || !git::is_clean(&winner.worktree).await?
9961 {
9962 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9963 return Ok(false);
9964 }
9965 let expected = self
9966 .state
9967 .pr_pushed_head
9968 .as_deref()
9969 .unwrap_or(&request.head)
9970 .to_owned();
9971 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9972 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9975 return Ok(false);
9976 }
9977 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9978 let refspec = format!("{after}:refs/heads/{public}");
9979 let push = git::git_raw(
9980 &winner.worktree,
9981 &["push", &lease, &self.state.config.merge.remote, &refspec],
9982 )
9983 .await;
9984 match push {
9985 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9986 Ok(out) => {
9987 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9988 return Ok(false);
9989 }
9990 Err(e) => {
9991 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9992 return Ok(false);
9993 }
9994 }
9995 self.state.merge_repairs[i].phase = Phase::Complete;
9996 self.state.status = RunStatus::Landing;
9997 self.state.event(
9998 "repair",
9999 "new review and gate completed; fresh owner merge decision required",
10000 );
10001 self.state.save()?;
10002 Ok(true)
10003 }
10004
10005 fn merge_repair_artifacts(&self) -> PathBuf {
10006 let artifacts = agent::artifacts_dir(&self.state.dir());
10007 match self
10008 .state
10009 .merge_repairs
10010 .last()
10011 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10012 {
10013 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10014 None => artifacts,
10015 }
10016 }
10017
10018 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10019 if let Some(e2e) = self.state.head_e2e.take() {
10020 self.state
10021 .verification_history
10022 .push(crate::run::VerificationEvidence {
10023 superseded_by: None,
10024 archived_at: Some(Timestamp::now()),
10025 fixes_spent: Some(self.state.gate_fixes.len()),
10026 gate_head: None,
10027 gate_ran: false,
10028 gate: Vec::new(),
10029 e2e: Some(e2e),
10030 });
10031 }
10032 self.state.reviews.clear();
10033 self.state.gate.clear();
10034 self.state.gate_ran = false;
10035 self.state.gate_head = None;
10036 self.state.gate_fixes.clear();
10037 self.state.gate_repair_e2e = None;
10038 self.state.pre_gate.clear();
10039 self.state.pre_gate_commit = None;
10040 self.state.contested_handoff = None;
10041 self.state.land_reruns = None;
10042 self.state.land_approval = None;
10043 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10044 self.state.status = RunStatus::Reviewing;
10045 self.state.save()
10046 }
10047
10048 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10049 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10050 self.state.merge_repairs[index].failure = Some(why.clone());
10051 self.state.status = RunStatus::Blocked;
10052 self.state.event("repair", why);
10053 self.state.save()
10054 }
10055
10056 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10060 if let Some(existing) = self.state.seats.get(key)
10061 && existing.agent == agent
10062 {
10063 return existing.clone();
10064 }
10065 let fresh = if self.state.seats.contains_key(key) {
10068 handover_seat(key, agent, self.state.next_seat_seed())
10069 } else {
10070 SeatState::new(key, agent, self.state.seed)
10071 };
10072 self.state.seats.insert(key.to_owned(), fresh.clone());
10073 fresh
10074 }
10075
10076 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10082 match self.state.seats.get(key) {
10083 Some(s) if s.agent != spec.id => {
10084 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10085 }
10086 _ => spec,
10087 }
10088 }
10089
10090 fn view(&self, c: &Candidate) -> CandidateView {
10092 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10093 .unwrap_or_default();
10094 let (patch, _) = blind::sanitize_patch(
10095 &format!("candidate {} patch", c.label),
10096 &raw,
10097 &self.state.config.blind,
10098 );
10099 CandidateView {
10100 label: c.label,
10101 branch: c.branch.clone(),
10102 summary: c.summary.clone(),
10103 stat: c.stat.clone(),
10104 patch,
10105 }
10106 }
10107
10108 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10110 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10111 prompt::judge(
10112 "(see above)",
10113 &views,
10114 self.roles.judges.len(),
10115 base_short,
10116 "en",
10117 )
10118 }
10119
10120 fn vote_prompt_full(
10126 &self,
10127 j: usize,
10128 labels: &[char],
10129 language: &str,
10130 candidates: &[Candidate],
10131 base_short: &str,
10132 ) -> String {
10133 let mut text = format!(
10134 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10135 prompt::final_vote(labels, language),
10136 self.state.instruction,
10137 self.candidate_block(candidates, base_short)
10138 );
10139 if let Some(own) = self
10140 .state
10141 .judgements
10142 .get(j)
10143 .filter(|r| !r.ranking.is_empty())
10144 {
10145 let reasons = own
10146 .reasons
10147 .iter()
10148 .map(|(k, v)| format!("- {k}: {v}"))
10149 .collect::<Vec<_>>()
10150 .join("\n");
10151 text.push_str(&format!(
10152 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10153 own.ranking.iter().collect::<String>(),
10154 if reasons.is_empty() {
10155 ""
10156 } else {
10157 ", because:\n"
10158 }
10159 ));
10160 }
10161 if !self.state.deliberation.is_empty() {
10162 text.push_str("\n# What was argued before this vote\n");
10163 for t in self.transcript(&[], j) {
10164 text.push_str(&format!(
10165 "\n## {}{}\n\n{}\n",
10166 t.who,
10167 if t.is_self { " (you)" } else { "" },
10168 t.body.trim()
10169 ));
10170 }
10171 }
10172 text
10173 }
10174
10175 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10182 let mut turns = Vec::new();
10183 for j in &self.state.judgements {
10184 if j.ranking.is_empty() {
10185 continue;
10186 }
10187 let reasons = j
10188 .reasons
10189 .iter()
10190 .map(|(k, v)| format!("- {k}: {v}"))
10191 .collect::<Vec<_>>()
10192 .join("\n");
10193 turns.push(Turn {
10194 who: format!("Judge {} (opening ranking)", j.judge),
10195 is_self: j.judge == self_idx + 1,
10196 body: format!(
10197 "Ranked {}{}{reasons}",
10198 j.ranking.iter().collect::<String>(),
10199 if reasons.is_empty() {
10200 ""
10201 } else {
10202 ", because:\n"
10203 }
10204 ),
10205 });
10206 }
10207 for t in self
10208 .state
10209 .deliberation
10210 .iter()
10211 .flat_map(|r| r.turns.iter())
10212 .chain(current)
10213 {
10214 turns.push(Turn {
10215 who: format!("Judge {}", t.judge),
10216 is_self: t.judge == self_idx + 1,
10217 body: t.body.clone(),
10218 });
10219 }
10220 turns
10221 }
10222}
10223
10224fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10226 agent::has_session(spec.kind, seat, sessions)
10227}
10228
10229fn next_untried_in_roster<'a>(
10250 roster: &'a [AgentSpec],
10251 start: usize,
10252 tried: &BTreeSet<String>,
10253) -> Option<&'a AgentSpec> {
10254 roster
10255 .get(start + 1..)?
10256 .iter()
10257 .find(|s| !tried.contains(&s.id))
10258}
10259
10260fn pick_successor<'a>(
10281 roster: &'a [AgentSpec],
10282 start: usize,
10283 tried: &BTreeSet<String>,
10284 carried: Option<&BTreeSet<String>>,
10285 others: &BTreeSet<String>,
10286) -> Option<&'a AgentSpec> {
10287 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10288 roster
10289 .get(start + 1..)
10290 .and_then(|tail| {
10291 tail.iter()
10292 .filter(free)
10293 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10294 })
10295 .or_else(|| {
10296 carried
10297 .is_some()
10298 .then(|| roster.iter().find(free))
10299 .flatten()
10300 })
10301 .or_else(|| match carried {
10302 Some(c) => next_for_seat(roster, start, tried, c),
10303 None => next_untried_in_roster(roster, start, tried),
10304 })
10305}
10306
10307fn next_for_seat<'a>(
10317 roster: &'a [AgentSpec],
10318 start: usize,
10319 round_tried: &BTreeSet<String>,
10320 carried_failed: &BTreeSet<String>,
10321) -> Option<&'a AgentSpec> {
10322 roster
10323 .get(start + 1..)?
10324 .iter()
10325 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10326 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10327}
10328
10329fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10335 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10336 return spec;
10337 };
10338 let ok = |id: &str| !h.failed.contains(id);
10339 if let Some(last) = h.last_ok.as_deref()
10340 && ok(last)
10341 && let Some(s) = roster.iter().find(|s| s.id == last)
10342 {
10343 return s.clone();
10344 }
10345 if ok(&spec.id) {
10346 return spec;
10347 }
10348 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10349 roster
10350 .get(from + 1..)
10351 .into_iter()
10352 .flatten()
10353 .chain(roster.iter())
10354 .find(|s| ok(&s.id))
10355 .cloned()
10356 .unwrap_or(spec)
10357}
10358
10359pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10363 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10364}
10365
10366const TIMED_OUT: &str = "timed out";
10369
10370impl FailClass {
10371 fn of(out: &AgentOutcome) -> Option<Self> {
10373 match out {
10374 AgentOutcome::Ok(_) => None,
10375 AgentOutcome::Quota(_) => Some(Self::Quota),
10376 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10377 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10378 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10379 AgentOutcome::ResourceBlocked(_) => None,
10382 }
10383 }
10384
10385 fn stem_word(&self) -> &'static str {
10387 match self {
10388 Self::Quota => "quota",
10389 _ => "handover",
10390 }
10391 }
10392}
10393
10394fn failure_signature(msg: &str) -> String {
10398 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10399 if let Some(code) = line
10402 .strip_prefix("process exited with no usable output (exit code ")
10403 .and_then(|s| s.strip_suffix(')'))
10404 .and_then(|s| s.parse::<i32>().ok())
10405 {
10406 line = format!("exited with some({code}) and no usable output");
10407 } else if matches!(
10408 line.as_str(),
10409 "process exited with no usable output (no exit code reported)"
10410 | "process exited with no usable output (exit code unavailable)"
10411 ) {
10412 line = "exited with none and no usable output".to_owned();
10413 }
10414 let mut out = Vec::new();
10415 for word in line.split_whitespace() {
10416 if word.contains('/') || word.contains('\\') {
10417 out.push("<path>".to_owned());
10418 continue;
10419 }
10420 let mut w = String::new();
10421 let mut in_digits = false;
10422 for c in word.chars() {
10423 if c.is_ascii_digit() {
10424 if !in_digits {
10425 w.push('#');
10426 }
10427 in_digits = true;
10428 } else {
10429 in_digits = false;
10430 w.push(c);
10431 }
10432 }
10433 out.push(w);
10434 }
10435 out.join(" ").chars().take(120).collect()
10436}
10437
10438fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10445 match cur {
10446 FailClass::Quota | FailClass::Timeout => true,
10447 FailClass::Other(_) => prev != Some(cur),
10448 }
10449}
10450
10451fn fail_reason(out: &AgentOutcome) -> String {
10453 match out {
10454 AgentOutcome::Ok(_) => String::new(),
10455 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10456 AgentOutcome::Dropped(o) => format!(
10457 "the CLI dropped the stream ({})",
10458 o.dropped
10459 .as_ref()
10460 .map(|d| d.why.as_str())
10461 .unwrap_or("it ended without delivering its answer")
10462 ),
10463 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10464 e.lines().next().unwrap_or("").chars().take(160).collect()
10465 }
10466 }
10467}
10468
10469pub(crate) fn record_handover(
10472 state: &mut RunState,
10473 node: &str,
10474 seat: &str,
10475 from: &str,
10476 to: &str,
10477 class: &FailClass,
10478 reason: &str,
10479) {
10480 let message = if *class == FailClass::Quota {
10481 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10482 } else {
10483 format!("{seat}: handed over {from} -> {to} ({reason})")
10484 };
10485 state.event(node, message);
10486 state.handovers.push(Handover {
10487 at: Timestamp::now(),
10488 node: node.to_owned(),
10489 seat: seat.to_owned(),
10490 from: from.to_owned(),
10491 to: to.to_owned(),
10492 reason: reason.to_owned(),
10493 });
10494}
10495
10496fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10510 commands.iter().any(|c| c.exit_code.is_none())
10511}
10512
10513fn verified_noop_claim(
10526 usable: bool,
10527 commands: &[agent::CommandEvidence],
10528 text: &str,
10529) -> Option<String> {
10530 (usable && !has_unconfirmed_command(commands))
10531 .then(|| verdict::verified_noop(text))
10532 .flatten()
10533}
10534
10535fn short(commit: &str) -> String {
10536 commit.chars().take(7).collect()
10537}
10538
10539fn make_executable(path: &Path) -> Result<()> {
10540 #[cfg(unix)]
10541 {
10542 use std::os::unix::fs::PermissionsExt as _;
10543 let mut perms = std::fs::metadata(path)?.permissions();
10544 perms.set_mode(0o755);
10545 std::fs::set_permissions(path, perms)?;
10546 }
10547 #[cfg(not(unix))]
10548 {
10549 let _ = path;
10550 }
10551 Ok(())
10552}
10553
10554struct WaveCtx<'a> {
10561 run: &'a str,
10564 node: &'a str,
10566 prompts: &'a Prompts,
10567 cache: Option<&'a Path>,
10569 round: Option<usize>,
10572 carry_seats: bool,
10576 pauses: Option<(&'a Pause, &'a Pause)>,
10579}
10580
10581fn repository_wait(
10583 state: &RunState,
10584 pauses: Option<(&Pause, &Pause)>,
10585) -> crate::repo_lock::WaitPolicy {
10586 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10587 let inherited = crate::repo_lock::WaitPolicy::current();
10588 crate::repo_lock::WaitPolicy::new(
10589 Duration::from_secs(state.config.graph.timeout_resource_wait),
10590 move || {
10591 inherited.cancelled()
10592 || pauses
10593 .as_ref()
10594 .is_some_and(|(a, b)| a.parked() || b.parked())
10595 },
10596 )
10597}
10598
10599fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10600 if error.is::<crate::repo_lock::ResourceBlocked>() {
10601 state.resource_block = Some(crate::run::ResourceBlock {
10602 cause: crate::run::ResourceBlockCause::Other,
10603 node: "repository".into(),
10604 kind: if error.to_string().contains("disk pressure") {
10605 "disk pressure"
10606 } else if error.to_string().contains("disk admission") {
10607 "disk admission"
10608 } else if error.to_string().contains(crate::incremental::BLOCKED) {
10609 "incremental cache admission"
10610 } else {
10611 "repository coordination"
10612 }
10613 .into(),
10614 detail: error.to_string(),
10615 at: Timestamp::now(),
10616 });
10617 state.event("resource", error.to_string());
10618 }
10619}
10620
10621async fn coordinate_operation(
10622 state: &mut RunState,
10623 cwd: &Path,
10624 wait: &crate::repo_lock::WaitPolicy,
10625) -> Result<Option<crate::repo_lock::SeatLease>> {
10626 let result =
10627 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10628 .await;
10629 if let Err(e) = &result {
10630 record_repository_block(state, e);
10631 }
10632 result
10633}
10634
10635async fn run_one(
10637 job: SeatJob,
10638 sem: Arc<Semaphore>,
10639 ctx: &WaveCtx<'_>,
10640 state: &mut RunState,
10641 attempt: usize,
10642) -> (SeatState, AgentOutcome) {
10643 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10644 .await
10645 .pop()
10646 .expect("one job in, one result out");
10647 (seat, out)
10648}
10649
10650async fn wave(
10656 jobs: Vec<SeatJob>,
10657 sem: Arc<Semaphore>,
10658 ctx: &WaveCtx<'_>,
10659 state: &mut RunState,
10660 attempt: usize,
10661) -> Vec<(usize, SeatState, AgentOutcome)> {
10662 let WaveCtx {
10663 run,
10664 node,
10665 prompts,
10666 cache,
10667 round,
10668 carry_seats: _,
10669 pauses,
10670 } = *ctx;
10671 for job in &jobs {
10672 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10673 }
10674 if let Err(e) = state.save() {
10675 tracing::warn!("could not persist in-progress seats: {e:#}");
10680 }
10681 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10697 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10701 let cache_guard = if let Some(cache_dir) = cache {
10702 if jobs_had_a_writer {
10703 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10704 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10705 match acquire_cache_lease(
10706 state,
10707 cache_dir,
10708 &owner,
10709 resource_limit,
10710 node,
10711 node,
10712 &cancel,
10713 )
10714 .await
10715 {
10716 Ok(Some(g)) => Some(g),
10717 Ok(None) => {
10720 if let Some(block) = &mut state.resource_block {
10721 block.cause = crate::run::ResourceBlockCause::Cancelled {
10722 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10723 };
10724 }
10725 return block_wave(
10726 jobs,
10727 state,
10728 node,
10729 attempt,
10730 "parked while waiting for the shared build cache",
10731 );
10732 }
10733 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10736 Err(e) => {
10737 let why = format!("{e:#}");
10741 state.event(
10742 "resource",
10743 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10744 );
10745 return block_wave(jobs, state, node, attempt, &why);
10746 }
10747 }
10748 } else {
10749 None
10750 }
10751 } else {
10752 None
10753 };
10754 state.seats_spawning(node, attempt);
10757 if let Err(e) = state.save() {
10758 tracing::warn!("could not persist the start of a wave: {e:#}");
10759 }
10760 let slot_limit = resource_limit
10764 + jobs
10765 .iter()
10766 .map(|j| j.timeout)
10767 .max()
10768 .unwrap_or_default()
10769 .saturating_mul(jobs.len() as u32);
10770 let mut set = tokio::task::JoinSet::new();
10771 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10774 let overlay = prompts.overlay(node);
10775 let repository_wait = repository_wait(state, pauses);
10776 for (i, mut job) in jobs.into_iter().enumerate() {
10777 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10778 if cache.is_some() {
10779 job.prompt.push('\n');
10780 job.prompt
10781 .push_str(&prompt::build_cache_note(node, job.allow_write));
10782 }
10783 let sem = Arc::clone(&sem);
10784 let run = run.to_owned();
10785 let node = node.to_owned();
10786 let attachments = if node == "implement" {
10789 state.attachments.clone()
10790 } else {
10791 Vec::new()
10792 };
10793 let cache = cache
10804 .filter(|_| job.allow_write && cache_guard.is_some())
10805 .map(Path::to_path_buf);
10806 let spawned_tx = spawned_tx.clone();
10807 let repository_wait = repository_wait.clone();
10808 let coordination = state.config.coordination.clone();
10809 let fetched_base = state.fetched_base.clone();
10810 set.spawn(async move {
10811 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10815 Ok(p) => p,
10816 Err(_) => {
10817 return (
10818 i,
10819 job.seat,
10820 AgentOutcome::ResourceBlocked(format!(
10821 "no parallel slot became free within {}s",
10822 slot_limit.as_secs()
10823 )),
10824 false,
10825 );
10826 }
10827 };
10828 spawned_tx.send(job.seat.key.clone()).ok();
10829 let mut seat = job.seat;
10830 let out = repository_wait
10831 .scope(agent::invoke(
10832 &job.spec,
10833 &mut seat,
10834 &Invocation {
10835 coordination: coordination.as_ref(),
10836 fetched_base: fetched_base.as_ref(),
10837 cwd: &job.cwd,
10838 prompt: &job.prompt,
10839 timeout: job.timeout,
10840 allow_write: job.allow_write,
10841 sessions: job.sessions,
10842 artifacts: &job.artifacts,
10843 stem: &job.stem,
10844 run: &run,
10845 node: &node,
10846 cache_dir: cache.as_deref(),
10847 attachments: &attachments,
10848 writable: &[],
10849 },
10850 ))
10851 .await;
10852 let interrupted = out
10853 .as_ref()
10854 .err()
10855 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10856 let out = match out {
10857 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10858 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10859 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10867 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10868 Ok(o) => AgentOutcome::Failed(format!(
10869 "process exited with no usable output ({})",
10870 crate::notices::exit_status(o.exit_code)
10871 )),
10872 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10873 AgentOutcome::ResourceBlocked(e.to_string())
10874 }
10875 Err(e) => AgentOutcome::Failed(e.to_string()),
10876 };
10877 (i, seat, out, interrupted)
10878 });
10879 }
10880 drop(spawned_tx);
10881 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10882 loop {
10883 let joined = tokio::select! {
10884 Some(key) = spawned_rx.recv() => {
10885 state.seat_spawned(&key);
10887 if let Err(e) = state.save() {
10888 tracing::warn!("could not persist a seat's start: {e:#}");
10889 }
10890 continue;
10891 }
10892 joined = set.join_next() => joined,
10893 };
10894 let Some(joined) = joined else {
10895 break;
10896 };
10897 let (i, seat, out, interrupted) = match joined {
10898 Ok(v) => v,
10899 Err(e) => {
10903 tracing::error!("agent task panicked: {e}");
10904 continue;
10905 }
10906 };
10907 state.seat_finished(&seat.key);
10908 if interrupted
10909 && node == "implement"
10910 && let Some(label) = state
10911 .candidates
10912 .iter()
10913 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10914 .map(|candidate| candidate.label)
10915 {
10916 state.candidate_preservation.insert(
10917 label,
10918 "interrupted implementation awaiting persistence".into(),
10919 );
10920 state.interrupted_implementations.insert(label);
10921 }
10922 if let AgentOutcome::ResourceBlocked(why) = &out
10923 && state.resource_block.is_none()
10924 {
10925 state.resource_block = Some(crate::run::ResourceBlock {
10926 cause: crate::run::ResourceBlockCause::Other,
10927 node: node.to_owned(),
10928 kind: if why.contains("disk pressure") {
10929 "disk pressure"
10930 } else if why.contains("disk admission") {
10931 "disk admission"
10932 } else if why.contains(crate::incremental::BLOCKED) {
10933 "incremental cache admission"
10934 } else if why.contains("repository coordination") {
10935 "repository coordination"
10936 } else {
10937 "parallel slot"
10938 }
10939 .to_owned(),
10940 detail: why.clone(),
10941 at: Timestamp::now(),
10942 });
10943 let activity = if interrupted {
10944 "agent execution interrupted"
10945 } else {
10946 "no agent was started"
10947 };
10948 state.event("resource", format!("{node}: {activity} - {why}"));
10949 }
10950 record_jobs(state, node, round, &seat.key, &out);
10951 if let Err(e) = state.save() {
10952 tracing::warn!("could not persist a seat's completion: {e:#}");
10953 }
10954 if collected.len() <= i {
10955 collected.resize_with(i + 1, || None);
10956 }
10957 collected[i] = Some((i, seat, out));
10958 }
10959 if state
10965 .active
10966 .values()
10967 .any(|a| a.node == node && a.attempt == attempt)
10968 {
10969 state
10970 .active
10971 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10972 if let Err(e) = state.save() {
10973 tracing::warn!("could not persist the end of a wave: {e:#}");
10974 }
10975 }
10976 if let Some(cache_dir) = cache
10983 && jobs_had_a_writer
10984 {
10985 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10986 }
10987 if let Some(guard) = cache_guard {
10988 guard.release();
10989 }
10990 collected.into_iter().flatten().collect()
10991}
10992
10993fn record_jobs(
11004 state: &mut RunState,
11005 node: &str,
11006 round: Option<usize>,
11007 seat: &str,
11008 out: &AgentOutcome,
11009) {
11010 let commands: &[agent::CommandEvidence] = match out {
11011 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11012 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11013 };
11014 let checked_at = Timestamp::now();
11015 for c in commands {
11016 state.jobs.push(JobRecord {
11017 node: node.to_owned(),
11018 round,
11019 seat: seat.to_owned(),
11020 id: c.id.clone(),
11021 description: c.description.clone(),
11022 checked_at,
11023 status: match c.exit_code {
11024 Some(0) => JobStatus::Completed,
11025 Some(_) => JobStatus::Failed,
11026 None => JobStatus::Unknown,
11027 },
11028 exit_code: c.exit_code,
11029 result_summary: c.result_summary.clone(),
11030 source: c.source.clone(),
11031 });
11032 }
11033}
11034
11035fn round_is_clean(
11056 blocking: usize,
11057 e2e_ok: bool,
11058 answered: usize,
11059 expected: usize,
11060 quota_missing: usize,
11061 policy: IncompleteReviewPolicy,
11062) -> bool {
11063 if blocking != 0 || !e2e_ok {
11064 return false;
11065 }
11066 if answered == expected || policy == IncompleteReviewPolicy::Warn {
11067 return true;
11068 }
11069 answered > 0 && expected - answered <= quota_missing
11070}
11071
11072pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11096 if max_rounds == 0
11097 || reviews.last().is_some_and(|r| {
11098 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11099 })
11100 {
11101 return Some(RunStatus::Gating);
11102 }
11103 let last = reviews.last()?;
11104 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11105 if reviews.len() < max_rounds && !stagnant {
11106 return None;
11107 }
11108 if last.incomplete() && last.blocking == 0 {
11109 return Some(RunStatus::Blocked);
11110 }
11111 if last.e2e_status() == E2eStatus::ResourceBlocked
11112 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11113 {
11114 return None;
11115 }
11116 Some(if last.e2e.iter().all(CommandOutcome::ok) {
11117 RunStatus::Gating
11118 } else {
11119 RunStatus::Blocked
11120 })
11121}
11122
11123pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11138 if nudged {
11139 (full / 4).max(Duration::from_secs(120)).min(full)
11140 } else {
11141 full
11142 }
11143}
11144
11145#[allow(clippy::too_many_arguments)]
11174async fn ask_json_wave<T>(
11175 jobs: Vec<SeatJob>,
11176 sem: Arc<Semaphore>,
11177 retries: usize,
11178 roster: &[AgentSpec],
11179 ctx: &WaveCtx<'_>,
11180 losses: &mut Vec<QuotaLoss>,
11181 state: &mut RunState,
11182 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11183) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11184where
11185 T: serde::de::DeserializeOwned + Send + 'static,
11186{
11187 ask_wave_with(
11188 jobs,
11189 sem,
11190 retries,
11191 roster,
11192 ctx,
11193 losses,
11194 state,
11195 &|text: &str| {
11196 let v = verdict::extract_json::<T>(text)?;
11197 validate(&v)?;
11198 Ok(v)
11199 },
11200 )
11201 .await
11202}
11203
11204#[allow(clippy::too_many_arguments)]
11213async fn ask_wave_with<T>(
11214 jobs: Vec<SeatJob>,
11215 sem: Arc<Semaphore>,
11216 retries: usize,
11217 roster: &[AgentSpec],
11218 ctx: &WaveCtx<'_>,
11219 losses: &mut Vec<QuotaLoss>,
11220 state: &mut RunState,
11221 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11222) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11223where
11224 T: Send + 'static,
11225{
11226 let n = jobs.len();
11227 let originals: Vec<SeatJob> = jobs;
11228 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11229 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11230 let mut nudges: Vec<usize> = vec![0; n];
11237 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11241 let mut tried: Vec<BTreeSet<String>> = specs
11242 .iter()
11243 .map(|s| BTreeSet::from([s.id.clone()]))
11244 .collect();
11245 let starts: Vec<usize> = specs
11246 .iter()
11247 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11248 .collect();
11249 let carry = ctx.carry_seats && !roster.is_empty();
11250 let carried: Vec<BTreeSet<String>> = originals
11253 .iter()
11254 .map(|j| {
11255 state
11256 .seat_history
11257 .get(&j.seat.key)
11258 .filter(|_| carry)
11259 .map(|h| h.failed.clone())
11260 .unwrap_or_default()
11261 })
11262 .collect();
11263 let mut prev: Vec<Option<FailClass>> = originals
11264 .iter()
11265 .map(|j| {
11266 state
11267 .seat_history
11268 .get(&j.seat.key)
11269 .filter(|_| carry)
11270 .and_then(|h| h.last_fail.clone())
11271 })
11272 .collect();
11273 let next_agent =
11274 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11275 let others: BTreeSet<String> = specs
11276 .iter()
11277 .enumerate()
11278 .filter(|(j, _)| *j != i)
11279 .map(|(_, s)| s.id.clone())
11280 .collect();
11281 pick_successor(
11282 roster,
11283 starts[i],
11284 tried,
11285 carry.then(|| &carried[i]),
11286 &others,
11287 )
11288 .cloned()
11289 };
11290 let mut fresh: Vec<Option<String>> = vec![None; n];
11291 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11292 let mut pending: Vec<usize> = (0..n).collect();
11293
11294 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11299 for round in 0..max_rounds {
11300 if pending.is_empty() {
11301 break;
11302 }
11303 let mut batch = Vec::with_capacity(pending.len());
11304 let mut renudged: Vec<&str> = Vec::new();
11305 for &i in &pending {
11306 let src = &originals[i];
11307 let full: &str = match &src.handover {
11312 Some(h) if specs[i].id != src.spec.id => h,
11313 _ => &src.prompt,
11314 };
11315 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11318 (
11319 full.to_owned(),
11320 src.timeout,
11321 format!("{}-{word}-{}", src.stem, specs[i].id),
11322 )
11323 } else if nudges[i] == 0 {
11324 (full.to_owned(), src.timeout, src.stem.clone())
11325 } else {
11326 renudged.push(src.seat.key.as_str());
11327 let why = done[i]
11328 .as_ref()
11329 .and_then(|r| r.as_ref().err().map(ToString::to_string))
11330 .unwrap_or_else(|| "no parsable answer".to_owned());
11331 let nudge = prompt::nudge(&why);
11332 let nudged = has_context(&specs[i], &seats[i], src.sessions);
11333 let prompt = if nudged {
11334 nudge
11335 } else {
11336 format!("{full}\n\n---\n\n{nudge}")
11337 };
11338 (
11339 prompt,
11340 retry_budget(src.timeout, nudged),
11341 format!("{}-retry{}", src.stem, nudges[i]),
11342 )
11343 };
11344 batch.push(SeatJob {
11345 spec: specs[i].clone(),
11346 seat: seats[i].clone(),
11347 cwd: src.cwd.clone(),
11348 prompt,
11349 timeout,
11350 allow_write: src.allow_write,
11351 sessions: src.sessions,
11352 artifacts: src.artifacts.clone(),
11353 stem,
11354 handover: None,
11355 });
11356 }
11357
11358 if !renudged.is_empty() {
11359 state.event(
11360 ctx.node,
11361 format!("retry {round}: re-asking {}", renudged.join(", ")),
11362 );
11363 }
11364 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11365 let mut still = Vec::new();
11366 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11367 seats[i] = seat;
11368 if let AgentOutcome::ResourceBlocked(why) = &out {
11371 done[i] = Some(Err(anyhow::anyhow!("{why}")));
11372 continue;
11373 }
11374 let class = FailClass::of(&out);
11375 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11378 && nudges[i] < retries
11379 && !roster.is_empty();
11380 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11381 let next = should_hand_over(prev[i].as_ref(), &cur)
11382 .then(|| next_agent(i, &tried[i], &specs))
11383 .flatten();
11384 if carry {
11385 let h = state
11386 .seat_history
11387 .entry(originals[i].seat.key.clone())
11388 .or_default();
11389 h.failed.insert(specs[i].id.clone());
11390 h.last_fail = Some(cur.clone());
11391 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11392 h.last_ok = None;
11393 }
11394 if let Err(e) = state.save() {
11397 tracing::warn!("could not persist a seat's failure history: {e:#}");
11398 }
11399 }
11400 if let Some(next) = next {
11401 record_handover(
11402 state,
11403 ctx.node,
11404 &originals[i].seat.key,
11405 &specs[i].id,
11406 &next.id,
11407 &cur,
11408 &fail_reason(&out),
11409 );
11410 tried[i].insert(next.id.clone());
11411 prev[i] = Some(cur.clone());
11412 seats[i] =
11413 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11414 specs[i] = next;
11415 fresh[i] = Some(cur.stem_word().to_owned());
11416 nudges[i] = 0;
11417 done[i] = Some(Err(anyhow::anyhow!(
11418 "handed over after: {}",
11419 fail_reason(&out)
11420 )));
11421 still.push(i);
11422 continue;
11423 }
11424 }
11425 if carry
11426 && !nudge_first
11427 && let Some(cur) = class.clone()
11428 {
11429 let h = state
11431 .seat_history
11432 .entry(originals[i].seat.key.clone())
11433 .or_default();
11434 h.failed.insert(specs[i].id.clone());
11435 h.last_fail = Some(cur);
11436 }
11437 let parsed = match out {
11438 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11439 AgentOutcome::Quota(o) => {
11440 last_quota[i] = o.quota.clone();
11441 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11442 }
11443 AgentOutcome::Dropped(o) => {
11447 let why = o
11448 .dropped
11449 .as_ref()
11450 .map(|d| d.why.as_str())
11451 .unwrap_or("the CLI ended the stream without delivering its answer");
11452 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11453 }
11454 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11455 Err(anyhow::anyhow!(e))
11456 }
11457 };
11458 let quota = class == Some(FailClass::Quota);
11459 let failed = parsed.is_err();
11460 if !failed && matches!(ctx.node, "judge" | "vote") {
11461 state
11464 .quota
11465 .retain(|loss| loss.seat != originals[i].seat.key);
11466 }
11467 done[i] = Some(parsed);
11468 if carry && !failed {
11469 let h = state
11470 .seat_history
11471 .entry(originals[i].seat.key.clone())
11472 .or_default();
11473 h.failed.remove(&specs[i].id);
11474 h.last_ok = Some(specs[i].id.clone());
11475 h.last_fail = None;
11476 }
11477 let agent_failure = class.is_some()
11485 && !nudge_first
11486 && !roster.is_empty()
11487 && next_agent(i, &tried[i], &specs).is_some();
11488 if failed && !quota && !agent_failure && nudges[i] < retries {
11489 nudges[i] += 1;
11490 still.push(i);
11491 }
11492 }
11493 pending = still;
11494 }
11495
11496 for (i, q) in last_quota.into_iter().enumerate() {
11499 if let Some(quota) = q {
11500 losses.push(QuotaLoss::from_report(
11501 originals[i].seat.key.clone(),
11502 ctx.node,
11503 Some("a),
11504 ));
11505 }
11506 }
11507
11508 if state.status != RunStatus::Stalled
11511 && ctx.node != "deliberate"
11512 && n > 0
11513 && losses.len() == n
11514 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11515 {
11516 block_machine_outage(state, ctx.node, losses);
11517 state.quota.append(losses);
11518 }
11519
11520 seats
11521 .into_iter()
11522 .zip(done)
11523 .zip(nudges)
11524 .map(|((seat, res), attempts)| {
11525 (
11526 seat,
11527 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11528 attempts,
11529 )
11530 })
11531 .collect()
11532}
11533
11534pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11537 let mut auth: Vec<_> = losses
11538 .iter()
11539 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11540 .map(|loss| loss.agent.clone())
11541 .collect();
11542 auth.sort();
11543 auth.dedup();
11544 let (kind, detail) = if auth.is_empty() {
11545 (
11546 "agent usage limit",
11547 "All seats reached their agent usage limits; resume after usage is available"
11548 .to_owned(),
11549 )
11550 } else {
11551 (
11552 "agent authentication",
11553 format!(
11554 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11555 auth.join(", ")
11556 ),
11557 )
11558 };
11559 state.resource_block = Some(crate::run::ResourceBlock {
11560 node: node.to_owned(),
11561 kind: kind.to_owned(),
11562 cause: crate::run::ResourceBlockCause::Other,
11563 detail: detail.clone(),
11564 at: Timestamp::now(),
11565 });
11566 state.event(node, detail);
11567}
11568
11569fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11571 results
11572 .iter()
11573 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11574}
11575
11576fn block_wave(
11580 jobs: Vec<SeatJob>,
11581 state: &mut RunState,
11582 node: &str,
11583 attempt: usize,
11584 why: &str,
11585) -> Vec<(usize, SeatState, AgentOutcome)> {
11586 state.resource_wait = None;
11587 state
11588 .active
11589 .retain(|_, a| !(a.node == node && a.attempt == attempt));
11590 if let Err(e) = state.save() {
11591 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11592 }
11593 jobs.into_iter()
11594 .enumerate()
11595 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11596 .collect()
11597}
11598
11599async fn acquire_cache_lease(
11612 state: &mut RunState,
11613 cache_dir: &Path,
11614 owner: &crate::cache::Owner,
11615 budget: Duration,
11616 node: &str,
11617 context: &str,
11618 cancel: &(dyn Fn() -> bool + Sync),
11619) -> Result<Option<crate::cache::Guard>> {
11620 let home = crate::run::home();
11621 let started = Instant::now();
11622 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11623 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11624 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11625 Err(e) => {
11626 state.event(
11627 "verify",
11628 format!("{context}: could not check the shared build cache: {e:#}"),
11629 );
11630 if let Err(e2) = state.save() {
11631 tracing::warn!("could not persist a cache-check failure: {e2:#}");
11632 }
11633 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11634 }
11635 };
11636 state.event(
11637 "verify",
11638 format!(
11639 "{context}: waiting for the shared build cache at {} ({})",
11640 cache_dir.display(),
11641 busy.describe()
11642 ),
11643 );
11644 state.resource_wait = Some(crate::run::ResourceWait {
11647 node: context.to_owned(),
11648 kind: "shared build cache".to_owned(),
11649 holder: busy.describe(),
11650 since: Timestamp::now(),
11651 limit_secs: budget.as_secs(),
11652 });
11653 if let Err(e) = state.save() {
11654 tracing::warn!("could not persist a cache wait: {e:#}");
11655 }
11656 let remaining = budget.saturating_sub(started.elapsed());
11657 let waited = crate::cache::wait_for_unless(
11658 &home,
11659 cache_dir,
11660 owner,
11661 remaining,
11662 Duration::from_secs(5),
11663 cancel,
11664 )
11665 .await;
11666 state.resource_wait = None;
11667 let cause = match &waited {
11668 Ok(Some(_)) => None,
11669 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11670 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11671 Some(crate::run::ResourceBlockCause::Timeout {
11672 waited_secs: started.elapsed().as_secs(),
11673 limit_secs: budget.as_secs(),
11674 })
11675 }
11676 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11677 };
11678 if let Some(cause) = cause {
11679 let detail = match &waited {
11680 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11681 Err(error) => format!("{error:#}"),
11682 };
11683 state.resource_block = Some(crate::run::ResourceBlock {
11684 cause,
11685 node: node.to_owned(),
11686 kind: "shared build cache".into(),
11687 detail,
11688 at: Timestamp::now(),
11689 });
11690 state.event(
11691 "resource",
11692 state.resource_block.as_ref().unwrap().description(),
11693 );
11694 }
11695 if let Err(error) = state.save() {
11696 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11697 }
11698 waited
11699}
11700
11701#[derive(Debug)]
11706struct CacheCheckError(String);
11707
11708impl std::fmt::Display for CacheCheckError {
11709 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11710 f.write_str(&self.0)
11711 }
11712}
11713
11714impl std::error::Error for CacheCheckError {}
11715
11716fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11718 state
11719 .resource_block
11720 .get_or_insert_with(|| crate::run::ResourceBlock {
11721 cause: crate::run::ResourceBlockCause::Other,
11722 node: node.to_owned(),
11723 kind: "shared build cache".into(),
11724 detail: format!("{error:#}"),
11725 at: Timestamp::now(),
11726 });
11727 if let Err(error) = state.save() {
11728 tracing::warn!("could not persist cache resource block: {error:#}");
11729 }
11730}
11731
11732#[allow(clippy::too_many_arguments)]
11753async fn with_cache_lease<'s, F, Fut>(
11754 state: &'s mut RunState,
11755 cache_dir: Option<&Path>,
11756 node: &str,
11757 seat: &str,
11758 worktree: &Path,
11759 head: &str,
11760 budget: Duration,
11761 context: &str,
11762 wait: &crate::repo_lock::WaitPolicy,
11763 body: F,
11764) -> Result<(Vec<CommandOutcome>, bool)>
11765where
11766 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11767 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11768{
11769 let started = Instant::now();
11772 let home = crate::run::home();
11773 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11774 let guard = if let Some(cache_dir) = cache_dir {
11775 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11776 match acquire_cache_lease(
11777 state,
11778 cache_dir,
11779 &owner,
11780 resource_limit,
11781 node,
11782 context,
11783 &|| wait.cancelled(),
11784 )
11785 .await
11786 .and_then(|g| {
11787 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11788 }) {
11789 Ok(g) => Some(g),
11790 Err(e) => {
11791 record_cache_error(state, node, &e);
11792 return Ok((
11793 vec![CommandOutcome {
11794 command: "(waiting for the shared build cache)".into(),
11795 code: None,
11796 output_tail: e.to_string(),
11797 duration_ms: started.elapsed().as_millis() as u64,
11798 resource_blocked: true,
11799 }],
11800 false,
11801 ));
11802 }
11803 }
11804 } else {
11805 None
11806 };
11807 let _coordination = coordinate_operation(state, worktree, wait).await?;
11808 if let Some(cache_dir) = cache_dir {
11809 let identity = crate::cache::Identity::new(worktree, head);
11810 if let Err(e) = crate::cache::ensure_fresh(
11811 &home,
11812 cache_dir,
11813 &identity,
11814 &state.config.verify.cargo_manifests(),
11815 ) {
11816 record_cache_error(state, node, &e);
11817 state.event(
11818 "verify",
11819 format!(
11820 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11821 worktree.display(),
11822 short(head)
11823 ),
11824 );
11825 if let Some(guard) = guard {
11826 guard.release();
11827 }
11828 return Ok((
11829 vec![CommandOutcome {
11830 command: "(confirming the shared build cache is fresh)".into(),
11831 code: None,
11832 output_tail: e.to_string(),
11833 duration_ms: started.elapsed().as_millis() as u64,
11834 resource_blocked: true,
11835 }],
11836 false,
11837 ));
11838 }
11839 }
11840 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11841 if !timed_out_pids.is_empty() {
11842 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11843 }
11844 drop(_coordination);
11847 if let Some(guard) = guard {
11848 guard.release();
11849 }
11850 Ok((outcomes, retried))
11851}
11852
11853async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11865 wait_for_pids_with(
11866 pids,
11867 crate::proc::pid_alive,
11868 LEASE_RELEASE_POLL,
11869 LEASE_RELEASE_MAX_WAIT,
11870 )
11871 .await;
11872}
11873
11874async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11880 pids: &[u32],
11881 alive: F,
11882 poll: Duration,
11883 max_wait: Duration,
11884) {
11885 let deadline = Instant::now() + max_wait;
11886 loop {
11887 if pids.iter().all(|&pid| !alive(pid)) {
11888 return;
11889 }
11890 if Instant::now() >= deadline {
11891 return;
11892 }
11893 tokio::time::sleep(poll).await;
11894 }
11895}
11896
11897fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11904 outcomes.iter().any(|o| o.resource_blocked)
11905}
11906
11907pub(crate) enum GateFix {
11909 Retry,
11911 Stop,
11914 Defer,
11917}
11918
11919pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11927 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11928 red.peek().is_some()
11929 && red.all(|o| {
11930 !o.resource_blocked
11931 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11932 && !o.output_tail.trim().is_empty()
11933 })
11934}
11935
11936fn e2e_outcome_label(o: &CommandOutcome) -> String {
11940 if o.ok() {
11941 return "pass".to_owned();
11942 }
11943 let reason = if o.build_failed() {
11944 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11945 } else {
11946 format!("FAIL ({:?})", o.code)
11947 };
11948 format!(
11949 "{reason}\n{}",
11950 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11951 )
11952}
11953
11954async fn run_e2e_with_retry(
11962 state: &mut RunState,
11963 shell: &[String],
11964 commands: &[String],
11965 worktree: &Path,
11966 timeout: Duration,
11967 context: &str,
11968) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11969 let (mut e2e, mut timed_out_pids) = run_commands(
11970 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11971 )
11972 .await;
11973 for o in &e2e {
11974 state.event(
11975 "verify",
11976 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11977 );
11978 }
11979 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11983 if verify_retried {
11984 state.event(
11985 "verify",
11986 format!(
11987 "{context}: verify could not build/link, not a test result — retrying once \
11988 before concluding"
11989 ),
11990 );
11991 let retried = run_commands(
11992 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11993 )
11994 .await;
11995 e2e = retried.0;
11996 timed_out_pids.extend(retried.1);
11999 for o in &e2e {
12000 state.event(
12001 "verify",
12002 format!(
12003 "{context}: retry `{}` -> {}",
12004 o.command,
12005 e2e_outcome_label(o)
12006 ),
12007 );
12008 }
12009 }
12010 (e2e, verify_retried, timed_out_pids)
12011}
12012
12013#[allow(clippy::too_many_arguments)]
12029async fn run_commands(
12030 state: &mut RunState,
12031 node: &str,
12032 task: &str,
12033 attempt: usize,
12034 shell: &[String],
12035 commands: &[String],
12036 cwd: &Path,
12037 timeout: Duration,
12038) -> (Vec<CommandOutcome>, Vec<u32>) {
12039 if commands.is_empty() {
12040 return (Vec::new(), Vec::new());
12045 }
12046 let mut out = Vec::new();
12047 let mut timed_out_pids = Vec::new();
12048 let total = commands.len();
12049 for (idx, command) in commands.iter().enumerate() {
12050 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12051 if let Err(e) = state.save() {
12052 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12053 }
12054 let started = Instant::now();
12055 let mut cmd = tokio::process::Command::new(&shell[0]);
12056 cmd.quiet();
12057 for key in crate::worker::INVOCATION_ENV {
12061 cmd.env_remove(key);
12062 }
12063 cmd.args(&shell[1..])
12064 .arg(command)
12065 .current_dir(cwd)
12066 .stdin(std::process::Stdio::null())
12067 .stdout(std::process::Stdio::piped())
12068 .stderr(std::process::Stdio::piped())
12069 .kill_on_drop(true);
12070 let mut outputs = Vec::new();
12071 let protected = (|| -> Result<()> {
12072 crate::incremental::configure(&mut cmd, cwd)?;
12073 let mut paths = vec![cwd.to_owned()];
12074 if let Some(cache) = state.config.cache_dir() {
12075 paths.push(cache);
12076 }
12077 outputs.extend(crate::retention::OutputLease::admit(
12078 &crate::run::home(),
12079 &paths,
12080 state.config.disk.min_free_bytes,
12081 )?);
12082 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12083 outputs.push(guard);
12084 }
12085 if let Some(cache) = state.config.cache_dir() {
12086 outputs.push(crate::retention::OutputLease::cache(
12087 &crate::run::home(),
12088 &cache,
12089 false,
12090 )?);
12091 }
12092 Ok(())
12093 })();
12094 if let Err(e) = protected {
12095 let detail = format!("execution admission unavailable: {e:#}");
12096 state.resource_block = Some(crate::run::ResourceBlock {
12097 cause: crate::run::ResourceBlockCause::Other,
12098 node: node.to_owned(),
12099 kind: if detail.contains("disk admission") {
12100 "disk admission"
12101 } else {
12102 "output ownership"
12103 }
12104 .to_owned(),
12105 detail: detail.clone(),
12106 at: Timestamp::now(),
12107 });
12108 out.push(CommandOutcome {
12109 command: command.clone(),
12110 code: None,
12111 output_tail: detail,
12112 duration_ms: 0,
12113 resource_blocked: true,
12114 });
12115 break;
12116 }
12117 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12118 let (code, body) = match spawned {
12119 Ok(mut running) => {
12120 let pid = running.child().id();
12121 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12122 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12123 let mut paths = vec![cwd.to_owned()];
12124 if let Some(cache) = state.config.cache_dir() {
12125 paths.push(cache);
12126 }
12127 let mut pressure = None;
12128 let status = tokio::select! {
12129 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12130 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12131 pressure = Some(reason); None
12132 }
12133 };
12134 let (code, failure) = match status {
12135 Some(Ok(Ok(status))) => (
12136 status.code(),
12137 status
12138 .code()
12139 .is_none()
12140 .then(|| format!("command terminated: {status}")),
12141 ),
12142 Some(Ok(Err(error))) => {
12143 (None, Some(format!("failed waiting for command: {error}")))
12144 }
12145 Some(Err(_)) => {
12146 if !running.coordinated()
12150 && let Some(pid) = pid
12151 {
12152 timed_out_pids.push(pid);
12153 }
12154 (
12155 None,
12156 Some(format!("timed out after {}s", timeout.as_secs())),
12157 )
12158 }
12159 None => (None, None),
12160 };
12161 running.finish().await;
12162 if let Some(reason) = pressure {
12163 state.resource_block = Some(crate::run::ResourceBlock {
12164 cause: crate::run::ResourceBlockCause::Other,
12165 node: node.to_owned(),
12166 kind: "disk pressure".to_owned(),
12167 detail: reason.clone(),
12168 at: Timestamp::now(),
12169 });
12170 out.push(CommandOutcome {
12171 command: command.clone(),
12172 code: None,
12173 output_tail: reason,
12174 duration_ms: started.elapsed().as_millis() as u64,
12175 resource_blocked: true,
12176 });
12177 break;
12178 }
12179 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12180 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12181 if let Some(failure) = failure {
12182 body.push('\n');
12183 body.push_str(&failure);
12184 }
12185 (code, body)
12186 }
12187 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12188 };
12189 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12190 if !body.is_empty() {
12191 match crate::run::write_verification_artifact(state, task, &body) {
12195 Ok(name) if summary != body => {
12196 summary = format!("[full output: artifacts/{name}]\n{summary}");
12197 }
12198 Ok(_) => {}
12199 Err(e) => {
12200 let warning = format!("could not save the full {task} output: {e:#}");
12201 tracing::warn!("{warning}");
12202 state.event(node, warning);
12203 if summary != body {
12204 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12205 }
12206 }
12207 }
12208 }
12209 let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12210 out.push(CommandOutcome {
12211 command: command.clone(),
12212 code,
12213 output_tail: summary,
12214 duration_ms: started.elapsed().as_millis() as u64,
12215 resource_blocked,
12216 });
12217 if resource_blocked {
12218 state.resource_block = Some(crate::run::ResourceBlock {
12219 cause: crate::run::ResourceBlockCause::Other,
12220 node: node.to_owned(),
12221 kind: "incremental cache admission".into(),
12222 detail: body,
12223 at: Timestamp::now(),
12224 });
12225 break;
12226 }
12227 }
12228 state.task_finished(task);
12229 if let Err(e) = state.save() {
12230 tracing::warn!("could not persist the end of {task}: {e:#}");
12231 }
12232 (out, timed_out_pids)
12233}
12234
12235fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12247 let repo = repo.display();
12248 match style {
12249 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12250 MergeStyle::Squash => {
12251 let subject = message
12254 .lines()
12255 .next()
12256 .unwrap_or(branch)
12257 .replace(['\\', '"', '$', '`'], "");
12258 format!(
12259 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12260 )
12261 }
12262 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12263 }
12264}
12265
12266const PR_TITLE_MAX: usize = 240;
12277
12278fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12289 let line = queue::first_line(text)?;
12290 let chars: Vec<char> = line.chars().collect();
12291 let (kept, mark) = if chars.len() <= max {
12292 (line.to_owned(), "")
12293 } else if let Some(end) = sentence_end(&chars, max) {
12294 (chars[..end].iter().collect::<String>(), "")
12295 } else {
12296 let room = max.saturating_sub(3);
12297 let cut = if chars[room].is_whitespace() {
12300 room
12301 } else {
12302 chars[..room]
12303 .iter()
12304 .rposition(|c| c.is_whitespace())
12305 .unwrap_or(room)
12306 };
12307 let head: String = chars[..cut].iter().collect();
12308 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12309 (head.to_owned(), "...")
12310 };
12311 if kept.is_empty() {
12312 return None;
12313 }
12314 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12315 return None;
12316 }
12317 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12318 return None;
12319 }
12320 Some(format!("{kept}{mark}"))
12321}
12322
12323fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12327 const MIN: usize = 20;
12328 for i in MIN..max.min(chars.len()) {
12329 if !matches!(chars[i], '.' | '!' | '?') {
12330 continue;
12331 }
12332 let Some(&next) = chars.get(i + 1) else {
12333 continue;
12334 };
12335 if !next.is_whitespace() {
12336 continue;
12337 }
12338 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12339 if after.is_some_and(|c| c.is_ascii_lowercase()) {
12340 continue;
12341 }
12342 let word: String = chars[..i]
12343 .iter()
12344 .rev()
12345 .take_while(|c| !c.is_whitespace())
12346 .collect::<Vec<_>>()
12347 .into_iter()
12348 .rev()
12349 .collect();
12350 let word = word.to_ascii_lowercase();
12351 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12352 continue;
12353 }
12354 let end = chars[..i]
12355 .iter()
12356 .rposition(|c| !c.is_whitespace())
12357 .map_or(i, |p| p + 1);
12358 return Some(end);
12359 }
12360 None
12361}
12362
12363struct PrMessage {
12368 title: String,
12369 body: String,
12370}
12371
12372impl PrMessage {
12373 fn commit_message(&self) -> String {
12377 format!("{}\n\n{}", self.title, self.body)
12378 }
12379}
12380
12381fn title_marker(line: &str) -> Option<&str> {
12383 let line = line.trim();
12384 let head = line.get(..6)?;
12385 head.eq_ignore_ascii_case("title:")
12386 .then(|| line[6..].trim())
12387}
12388
12389fn summary_title(summary: &str) -> Option<String> {
12394 let first = summary.lines().find(|l| !l.trim().is_empty())?;
12395 let raw = title_marker(first)?;
12396 if raw.is_empty() {
12397 return None;
12398 }
12399 let title = queue::title_from(raw, PR_TITLE_MAX);
12400 let lower = title.to_ascii_lowercase();
12401 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12402 return None;
12403 }
12404 Some(title)
12405}
12406
12407const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12409
12410pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12412
12413pub(crate) fn is_review_run(state: &RunState) -> bool {
12417 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12418}
12419
12420fn review_title(state: &RunState) -> Option<String> {
12425 english_subject(
12426 state.reviewed_commits.as_ref()?.first()?,
12427 state.config.graph.github_english(),
12428 )
12429}
12430
12431fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12434 let raw = raw.trim();
12435 if raw.is_empty() {
12436 return None;
12437 }
12438 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12439 return None;
12440 }
12441 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12442 return None;
12443 }
12444 let title = queue::title_from(raw, PR_TITLE_MAX);
12445 let lower = title.to_ascii_lowercase();
12446 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12447 return None;
12448 }
12449 Some(title)
12450}
12451
12452#[derive(Debug, Clone, PartialEq, Eq)]
12455struct BranchFacts {
12456 commits: Vec<(String, String)>,
12458 stat: String,
12460}
12461
12462const STAT_FILE_LINES: usize = 25;
12464const COMMITS_MAX_CHARS: usize = 20_000;
12466
12467async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12470 let commits = git::commit_log(repo, base, branch).await.ok()?;
12471 if commits.is_empty() {
12472 return None;
12473 }
12474 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12475 let lines: Vec<&str> = stat.lines().collect();
12476 let stat = if lines.len() > STAT_FILE_LINES + 1 {
12477 let omitted = lines.len() - 1 - STAT_FILE_LINES;
12478 let more = format!(" ... {omitted} more file(s)");
12479 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12480 kept.push(&more);
12481 kept.push(lines[lines.len() - 1]);
12482 kept.join("\n")
12483 } else {
12484 lines.join("\n")
12485 };
12486 Some(BranchFacts { commits, stat })
12487}
12488
12489fn markdown_safe(text: &str) -> String {
12492 text.replace("</details>", "</details>")
12493 .replace("\x60\x60\x60", "~~~")
12494}
12495
12496fn neutral_title(state: &RunState, winner: char) -> String {
12497 format!(
12498 "chore: land candidate {} of run {}",
12499 winner.to_ascii_uppercase(),
12500 state.id
12501 )
12502}
12503
12504pub(crate) async fn repair_landing_gate(
12507 state: &mut RunState,
12508 winner: &Candidate,
12509 outcomes: &[CommandOutcome],
12510) -> Result<GateFix> {
12511 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12512 return Ok(GateFix::Stop);
12513 }
12514 let mut runner = Runner {
12515 roles: state.config.resolve_roles()?,
12516 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12517 state: state.clone(),
12518 pause: Pause::new(),
12519 interrupt: Pause::new(),
12520 };
12521 let result = runner.gate_fix_round(winner, outcomes).await;
12522 *state = runner.state;
12523 result
12524}
12525
12526pub(crate) async fn verify_current_head(
12529 state: &mut RunState,
12530 winner: &Candidate,
12531 wait: &crate::repo_lock::WaitPolicy,
12532) -> Result<bool> {
12533 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12534 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12536 state
12537 .verification_history
12538 .push(crate::run::VerificationEvidence {
12539 superseded_by: Some(head.clone()),
12540 archived_at: Some(Timestamp::now()),
12541 fixes_spent: Some(state.gate_fixes.len()),
12542 gate_head: None,
12543 gate_ran: false,
12544 gate: Vec::new(),
12545 e2e: state.head_e2e.take(),
12546 });
12547 }
12548 if let Some(e) = &state.head_e2e
12549 && e.head == head
12550 {
12551 if e.passed {
12552 return Ok(true);
12553 }
12554 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12555 state.status = RunStatus::Blocked;
12556 state.save()?;
12557 return Ok(false);
12558 }
12559 } else if state.reviews.last().is_some_and(|r| {
12560 r.verified_head.as_deref() == Some(head.as_str())
12561 && (r.e2e_status() == E2eStatus::Passed
12562 || (r.e2e_status() == E2eStatus::NotConfigured
12563 && state.config.verify.e2e.is_empty()))
12564 }) || state
12565 .gate_repair_e2e
12566 .as_ref()
12567 .is_some_and(|e| e.head == head && e.passed)
12568 {
12569 return Ok(true);
12570 }
12571 if let Some(e) = &state.gate_repair_e2e
12572 && e.head == head
12573 && !e.passed
12574 && !e.outcomes.is_empty()
12575 && !verify_inconclusive(&e.outcomes)
12576 {
12577 state.status = RunStatus::Blocked;
12578 state.save()?;
12579 return Ok(false);
12580 }
12581 state.head_e2e = Some(crate::run::GateRepairE2e {
12582 head: head.clone(),
12583 passed: false,
12584 outcomes: Vec::new(),
12585 });
12586 state.save()?;
12587 let commands = state.config.verify.e2e.clone();
12588 let shell = state.config.shell();
12589 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12590 let cache_dir = state.config.cache_dir();
12591 let outcomes = if commands.is_empty() {
12592 Vec::new()
12593 } else {
12594 let (outcomes, _) = with_cache_lease(
12595 state,
12596 cache_dir.as_deref(),
12597 "e2e",
12598 "e2e",
12599 &winner.worktree,
12600 &head,
12601 timeout,
12602 "current head e2e",
12603 wait,
12604 |state, budget| async move {
12605 run_e2e_with_retry(
12606 state,
12607 &shell,
12608 &commands,
12609 &winner.worktree,
12610 budget,
12611 "current head e2e",
12612 )
12613 .await
12614 },
12615 )
12616 .await?;
12617 outcomes
12618 };
12619 let unresolved = verify_inconclusive(&outcomes);
12620 let passed = !unresolved
12621 && outcomes.iter().all(CommandOutcome::ok)
12622 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12623 if !outcomes.is_empty() && !unresolved {
12624 state.record_base_progress(&head);
12625 }
12626 state.head_e2e = Some(crate::run::GateRepairE2e {
12627 head: head.clone(),
12628 passed,
12629 outcomes,
12630 });
12631 if !passed && !unresolved {
12632 state.status = RunStatus::Blocked;
12633 state.event(
12634 "e2e",
12635 format!("verify.e2e failed on head {head}; nothing published"),
12636 );
12637 }
12638 state.save()?;
12639 Ok(passed)
12640}
12641
12642pub(crate) async fn check_gate_repair_e2e(
12645 state: &mut RunState,
12646 winner: &Candidate,
12647 wait: &crate::repo_lock::WaitPolicy,
12648) -> Result<GateFix> {
12649 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12650 return Ok(GateFix::Retry);
12651 }
12652 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12653 if state
12654 .gate_repair_e2e
12655 .as_ref()
12656 .is_some_and(|r| r.head == head && r.passed)
12657 {
12658 return Ok(GateFix::Retry);
12659 }
12660 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12661 head: head.clone(),
12662 passed: false,
12663 outcomes: Vec::new(),
12664 });
12665 state.save()?;
12666 let commands = state.config.verify.e2e.clone();
12667 let outcomes = if commands.is_empty() {
12668 Vec::new()
12669 } else {
12670 let shell = state.config.shell();
12671 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12672 let cache_dir = state.config.cache_dir();
12673 let (outcomes, _) = with_cache_lease(
12674 state,
12675 cache_dir.as_deref(),
12676 "e2e",
12677 "e2e",
12678 &winner.worktree,
12679 &head,
12680 timeout,
12681 "gate repair e2e",
12682 wait,
12683 |state, budget| async move {
12684 run_e2e_with_retry(
12685 state,
12686 &shell,
12687 &commands,
12688 &winner.worktree,
12689 budget,
12690 "gate repair e2e",
12691 )
12692 .await
12693 },
12694 )
12695 .await?;
12696 outcomes
12697 };
12698 let deferred = verify_inconclusive(&outcomes);
12699 let passed = outcomes.iter().all(CommandOutcome::ok)
12700 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12701 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12702 head,
12703 passed,
12704 outcomes,
12705 });
12706 state.save()?;
12707 if deferred {
12708 Ok(GateFix::Defer)
12709 } else if !passed {
12710 state.event(
12711 "gate",
12712 "verify.e2e failed after gate repair; nothing published",
12713 );
12714 Ok(GateFix::Stop)
12715 } else {
12716 Ok(GateFix::Retry)
12717 }
12718}
12719
12720pub(crate) async fn run_landing_gate(
12722 state: &mut RunState,
12723 winner: &Candidate,
12724 wait: &crate::repo_lock::WaitPolicy,
12725) -> Result<Vec<CommandOutcome>> {
12726 let shell = state.config.shell();
12727 let gate_commands = state.config.verify.gate.clone();
12728 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12729 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12730 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12731 Vec::new()
12732 } else {
12733 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12734 let cache_dir = state.config.cache_dir();
12735 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12736 let (outcomes, _) = with_cache_lease(
12737 state,
12738 cache_dir.as_deref(),
12739 "gate",
12740 "gate",
12741 &winner.worktree,
12742 &head,
12743 timeout,
12744 "final gate",
12745 wait,
12746 |state, budget| {
12747 let shell = shell.clone();
12748 let gate_commands = gate_commands.clone();
12749 let winner = winner.clone();
12750 async move {
12751 Runner::run_pre_gate(state, &winner).await;
12752 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12754 let (outcomes, timed_out_pids) = run_commands(
12755 state,
12756 "gate",
12757 "gate",
12758 0,
12759 &shell,
12760 &gate_commands,
12761 &winner.worktree,
12762 budget,
12763 )
12764 .await;
12765 (outcomes, false, timed_out_pids)
12766 }
12767 },
12768 )
12769 .await?;
12770 outcomes
12771 };
12772 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12773 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12774 let mut outcomes = outcomes;
12775 outcomes.push(CommandOutcome {
12776 command: "gate head stability".into(), code: None,
12777 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12778 duration_ms: 0, resource_blocked: false,
12779 });
12780 return Ok(outcomes);
12781 }
12782 if outcomes.is_empty() {
12783 state.event("gate", "no gate commands configured; not tested");
12788 }
12789 for o in &outcomes {
12790 state.event(
12791 "gate",
12792 format!(
12793 "`{}` -> {}",
12794 o.command,
12795 if o.ok() {
12796 "pass".to_owned()
12797 } else {
12798 format!(
12799 "FAIL ({:?})\n{}",
12800 o.code,
12801 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12802 )
12803 }
12804 ),
12805 );
12806 }
12807 Ok(outcomes)
12808}
12809
12810pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12816 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12817 ""
12818 } else {
12819 pr_title
12820 }
12821}
12822
12823pub fn landing_subject_source(state: &RunState) -> String {
12827 if is_review_run(state) {
12828 let winner = state.candidates.first().map_or('A', |c| c.label);
12829 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12830 }
12831 state.instruction.clone()
12832}
12833
12834fn summary_without_title(summary: &str) -> String {
12837 let mut lines = summary.trim().lines().peekable();
12838 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12839 lines.next();
12840 }
12841 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12842}
12843
12844#[cfg(test)]
12856fn pr_message(state: &RunState, winner: char) -> PrMessage {
12857 pr_message_with(state, winner, None)
12858}
12859
12860#[cfg(test)]
12863fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12864 let raw = pr_message_raw(
12865 state,
12866 winner,
12867 facts,
12868 crate::pr_outcome::FinalHead::Unreported,
12869 ChangeNote::NotAsked,
12870 );
12871 let id = crate::scrub::Identity::current();
12872 PrMessage {
12873 title: crate::scrub::scrub(&raw.title, &id),
12874 body: crate::scrub::scrub(&raw.body, &id),
12875 }
12876}
12877
12878#[derive(Debug, Clone, Copy)]
12880enum ChangeNote<'a> {
12881 NotAsked,
12883 Summary(&'a crate::run::ChangeSummary),
12885 Unavailable(&'a str),
12887}
12888
12889fn pr_message_raw(
12890 state: &RunState,
12891 winner: char,
12892 facts: Option<&BranchFacts>,
12893 head: crate::pr_outcome::FinalHead<'_>,
12894 change: ChangeNote<'_>,
12895) -> PrMessage {
12896 if !matches!(change, ChangeNote::NotAsked) {
12897 let (title, mut body) = match change {
12898 ChangeNote::Summary(c) => (
12899 c.title.clone(),
12900 format!("## Summary\n\n{}\n\n", c.body.trim()),
12901 ),
12902 ChangeNote::Unavailable(why) => (
12903 neutral_title(state, winner),
12904 format!(
12905 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12906 ),
12907 ),
12908 ChangeNote::NotAsked => unreachable!(),
12909 };
12910 body.push_str(&public_pr_evidence(state, head));
12911 body.push_str(&format!(
12912 "\n---\nmagi:run/{} magi:candidate-{}\n",
12913 state.id,
12914 winner.to_ascii_lowercase()
12915 ));
12916 return PrMessage { title, body };
12917 }
12918 pr_preview(state, winner, facts, head)
12919}
12920
12921fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12924 let mut body = crate::pr_outcome::review_rounds(state);
12925 body.push_str(&crate::pr_outcome::final_verification(state, head));
12926 let open = state.last_round_findings();
12927 if !open.is_empty() {
12928 body.push_str("## Open review findings\n\n");
12929 for f in open {
12930 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12931 }
12932 body.push('\n');
12933 }
12934 if let Some(c) = state.change_summary.as_ref()
12935 && head == crate::pr_outcome::FinalHead::At(&c.head)
12936 {
12937 body.push_str(&format!(
12938 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12939 c.base, c.head
12940 ));
12941 }
12942 body
12943}
12944
12945fn pr_preview(
12947 state: &RunState,
12948 winner: char,
12949 facts: Option<&BranchFacts>,
12950 head: crate::pr_outcome::FinalHead<'_>,
12951) -> PrMessage {
12952 let summary = state
12953 .candidates
12954 .iter()
12955 .find(|c| c.label == winner)
12956 .map(|c| c.summary.as_str())
12957 .unwrap_or_default();
12958 let review = is_review_run(state);
12961 let title = if review {
12962 facts
12963 .and_then(|f| {
12964 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12965 })
12966 .or_else(|| review_title(state))
12967 .or_else(|| {
12968 state
12969 .candidates
12970 .iter()
12971 .find(|c| c.label == winner)
12972 .filter(|c| !c.branch.starts_with("magi/"))
12973 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12974 })
12975 .unwrap_or_else(|| neutral_title(state, winner))
12976 } else {
12977 summary_title(summary).unwrap_or_else(|| {
12978 english_title(
12979 &state.instruction,
12980 PR_TITLE_MAX,
12981 state.config.graph.github_english(),
12982 )
12983 .unwrap_or_else(|| neutral_title(state, winner))
12984 })
12985 };
12986
12987 let mut body = String::new();
12988 let what = summary_without_title(summary);
12989 if !what.is_empty() {
12990 body.push_str("## Summary\n\n");
12991 body.push_str(&what);
12992 body.push_str("\n\n");
12993 }
12994
12995 body.push_str(&crate::pr_outcome::review_rounds(state));
12997 body.push_str(&crate::pr_outcome::final_verification(state, head));
12998 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12999
13000 let open = state.open_findings();
13001 if !open.is_empty() {
13002 body.push_str("## Open review findings\n\n");
13003 for f in &open {
13004 body.push_str(&format!(
13005 "- `{}` [{:?}] {}\n",
13006 f.id, f.severity, "Context retained in local review records."
13007 ));
13008 }
13009 body.push('\n');
13010 }
13011
13012 if let Some(fix) = fix
13013 && !fix.rejected.is_empty()
13014 {
13015 body.push_str("## Declined by the fixer\n\n");
13016 for r in &fix.rejected {
13017 body.push_str(&format!(
13018 "- `{}`: {}\n",
13019 r.id, "Reason retained in local review records."
13020 ));
13021 }
13022 body.push('\n');
13023 }
13024
13025 if review {
13026 body.push_str("## Commits under review\n\n");
13028 if let Some(facts) = facts {
13029 let mut left = COMMITS_MAX_CHARS;
13030 for (i, (subject, text)) in facts.commits.iter().enumerate() {
13031 let mut entry = format!("- {}\n", markdown_safe(subject));
13032 for l in markdown_safe(text).lines() {
13033 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
13034 }
13035 if left == 0 {
13036 body.push_str(&format!(
13037 "- ... {} more commit(s)\n",
13038 facts.commits.len() - i
13039 ));
13040 break;
13041 }
13042 if entry.len() > left {
13043 let mut end = left;
13046 while !entry.is_char_boundary(end) {
13047 end -= 1;
13048 }
13049 entry.truncate(end);
13050 entry.push_str("\n ... (truncated)\n");
13051 left = 0;
13052 } else {
13053 left -= entry.len();
13054 }
13055 body.push_str(&entry);
13056 }
13057 if !facts.stat.trim().is_empty() {
13058 body.push_str(&format!(
13059 "\n## Diff stat\n\n```\n{}\n```\n",
13060 markdown_safe(facts.stat.trim())
13061 ));
13062 }
13063 } else {
13064 match &state.reviewed_commits {
13065 Some(subjects) => {
13066 for s in subjects {
13067 body.push_str(&format!("- {}\n", s.trim()));
13068 }
13069 }
13070 None => {
13071 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13074 body.push_str(rest.trim());
13075 body.push('\n');
13076 }
13077 }
13078 }
13079 } else {
13080 let task = state.instruction.trim();
13081 let task = if task.is_empty() {
13082 "(empty task)"
13083 } else {
13084 task
13085 };
13086 body.push_str(&format!(
13087 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13088 task.replace("</details>", "</details>")
13089 ));
13090 }
13091
13092 body.push_str(&format!(
13093 "\n---\nmagi:run/{} magi:candidate-{}\n",
13094 state.id,
13095 winner.to_ascii_lowercase()
13096 ));
13097
13098 PrMessage { title, body }
13099}
13100
13101fn seeded_instruction(state: &RunState) -> String {
13104 let mut inputs = Vec::new();
13105 if let Some(start) = &state.sources.start_from {
13106 inputs.push(format!(
13107 "Start from `{}` at `{}` (including ancestry).",
13108 start.token, start.sha
13109 ));
13110 }
13111 for pick in &state.sources.pick_commits {
13112 inputs.push(format!(
13113 "Pick `{}` at `{}` (single commit).",
13114 pick.token, pick.sha
13115 ));
13116 }
13117 if inputs.is_empty() {
13118 state.instruction.clone()
13119 } else {
13120 format!(
13121 "{}\n\n# Explicit source inputs already prepared\n\n{}",
13122 state.instruction,
13123 inputs.join("\n")
13124 )
13125 }
13126}
13127
13128async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13132 let base = &state.base_branch;
13133 let mut against = base.clone();
13134 if mode == MergeMode::Pr {
13135 let remote = &state.config.merge.remote;
13139 let tracking = format!("{remote}/{base}");
13140 let fetched = git::fetch(repo, remote, base).await;
13141 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13142 against = tracking;
13143 } else {
13144 return false;
13145 }
13146 }
13147 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13148}
13149
13150fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13153 let mut detail = format!(
13154 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13155 nothing to open a pull request for"
13156 );
13157 detail.push_str(
13158 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13159 );
13160 detail
13161}
13162
13163#[derive(Debug, PartialEq, Eq)]
13166enum PrPlan {
13167 Create,
13168 Adopt { url: String, title: String },
13169 Stop(String),
13170}
13171
13172fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13175 match found {
13176 Ok(land::OpenPr::None) => PrPlan::Create,
13177 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13178 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13179 "several open pull requests exist for this branch, not picking one: {}",
13180 urls.join(" ")
13181 )),
13182 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13183 }
13184}
13185
13186pub(crate) async fn gh_pr_create(
13188 cwd: &Path,
13189 base: &str,
13190 head: &str,
13191 title: &str,
13192 body: &str,
13193) -> Result<String> {
13194 let out = tokio::process::Command::new("gh")
13195 .args([
13196 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13197 ])
13198 .current_dir(cwd)
13199 .quiet()
13200 .stdin(std::process::Stdio::null())
13201 .output()
13202 .await
13203 .context("spawn gh")?;
13204 if out.status.success() {
13205 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13206 } else {
13207 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13208 }
13209}
13210
13211async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13213 match std::fs::symlink_metadata(path) {
13214 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13215 git::worktree_remove_clean(repo, path).await?;
13218 return Ok(());
13219 }
13220 Err(e) => return Err(e.into()),
13221 Ok(_) => {}
13222 }
13223 if !git::is_clean(path).await? {
13224 bail!(
13225 "worktree has uncommitted work; refusing removal: {}",
13226 path.display()
13227 );
13228 }
13229 if !git::worktree_remove_clean(repo, path).await? {
13230 bail!("worktree removal refused: {}", path.display());
13231 }
13232 Ok(())
13233}
13234
13235pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13244 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13245 bail!("refusing fold: {reason}");
13246 }
13247 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13248 bail!("candidate preservation incomplete; refusing cleanup");
13249 }
13250 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13254 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13255 let _driver = crate::driver::maintenance(home, &state.id)?;
13256 let mut output_guards = Vec::new();
13257 for entry in std::fs::read_dir(state.worktree_root())
13258 .into_iter()
13259 .flatten()
13260 .flatten()
13261 {
13262 if entry.path().join(".git").is_file() {
13263 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13264 }
13265 }
13266 let repo = state.repo.clone();
13267 let wait = repository_wait(state, None);
13270 let _coordination = match coordinate_operation(state, &repo, &wait).await {
13271 Ok(lease) => lease,
13272 Err(e) => {
13273 state.save_under(home)?;
13274 return Err(e);
13275 }
13276 };
13277 let _source_outputs = if state.status == RunStatus::Merged
13280 && state.merge.as_ref().is_some_and(|merge| merge.ok)
13281 {
13282 verify_merged_fold_sources(state, drop_winner, home).await?
13283 } else {
13284 Vec::new()
13285 };
13286 let root = state.worktree_root();
13287 let winner = state.tally.as_ref().map(|t| t.winner);
13288 let mut removed = Vec::new();
13289
13290 for i in 0..state.candidates.len() {
13291 let c = state.candidates[i].clone();
13292 let is_winner = Some(c.label) == winner;
13293 if is_winner && !drop_winner {
13294 continue;
13295 }
13296 let existed = c.worktree.exists();
13297 remove_preserved_worktree(&repo, &c.worktree).await?;
13300 if existed {
13301 removed.push(c.worktree.to_string_lossy().into_owned());
13302 }
13303 let handed_over = state.released_branches.contains(&c.branch);
13306 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13307 if !git::branch_delete(&repo, &c.branch).await? {
13308 bail!("candidate branch removal refused");
13309 }
13310 removed.push(c.branch.clone());
13311 }
13312 state.candidates[i].folded = true;
13313 }
13314
13315 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13316 let path = name.path();
13317 let keep = !drop_winner
13318 && winner.is_some_and(|w| {
13319 path.file_name()
13320 .is_some_and(|n| n == format!("cand-{w}").as_str())
13321 });
13322 if keep {
13323 continue;
13324 }
13325 remove_preserved_worktree(&repo, &path).await?;
13326 removed.push(path.to_string_lossy().into_owned());
13327 }
13328
13329 remove_if_empty(&root);
13338
13339 if state.enabled_worktree_config && drop_winner {
13340 git::release_worktree_config(&repo).await.ok();
13344 state.enabled_worktree_config = false;
13345 }
13346 state.save_under(home)?;
13347 Ok(removed)
13348}
13349
13350async fn verify_merged_fold_sources(
13351 state: &RunState,
13352 drop_winner: bool,
13353 home: &Path,
13354) -> Result<Vec<std::fs::File>> {
13355 let winner = state.tally.as_ref().map(|t| t.winner);
13356 let mut heads = BTreeSet::new();
13357 for candidate in &state.candidates {
13358 if Some(candidate.label) == winner && !drop_winner {
13359 continue;
13360 }
13361 anyhow::ensure!(
13362 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13363 "candidate recovery source is outside the inventoried run root: {}",
13364 candidate.worktree.display()
13365 );
13366 match std::fs::symlink_metadata(&candidate.worktree) {
13367 Ok(_) => {}
13368 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13369 Err(error) => return Err(error.into()),
13370 }
13371 if !state.released_branches.contains(&candidate.branch)
13372 && git::branch_exists(&state.repo, &candidate.branch).await?
13373 {
13374 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13375 }
13376 }
13377 let mut cargo_guards = Vec::new();
13378 let entries = match std::fs::read_dir(state.worktree_root()) {
13379 Ok(entries) => Some(entries),
13380 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13381 Err(error) => return Err(error.into()),
13382 };
13383 for entry in entries.into_iter().flatten() {
13384 let path = entry?.path();
13385 let metadata = std::fs::symlink_metadata(&path)?;
13386 anyhow::ensure!(
13387 metadata.is_dir() && !metadata.file_type().is_symlink(),
13388 "non-directory or symlink recovery source retained: {}",
13389 path.display()
13390 );
13391 if !drop_winner
13392 && winner.is_some_and(|w| {
13393 path.file_name()
13394 .is_some_and(|name| name == format!("cand-{w}").as_str())
13395 })
13396 {
13397 continue;
13398 }
13399 anyhow::ensure!(
13400 path.join(".git").is_file(),
13401 "unregistered recovery source retained: {}",
13402 path.display()
13403 );
13404 anyhow::ensure!(
13405 git::git(
13406 &path,
13407 &[
13408 "status",
13409 "--porcelain",
13410 "--untracked-files=all",
13411 "--ignored",
13412 "--",
13413 ".",
13414 ":(exclude)target"
13415 ]
13416 )
13417 .await?
13418 .is_empty(),
13419 "source changes or ignored recovery files retained: {}",
13420 path.display()
13421 );
13422 match std::fs::symlink_metadata(path.join("target")) {
13423 Ok(_) => {
13424 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13425 }
13426 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13427 Err(error) => return Err(error.into()),
13428 }
13429 for checkpoint in [
13430 "rebase-merge",
13431 "rebase-apply",
13432 "sequencer",
13433 "CHERRY_PICK_HEAD",
13434 "MERGE_HEAD",
13435 "REBASE_HEAD",
13436 "REVERT_HEAD",
13437 "BISECT_LOG",
13438 "AUTO_MERGE",
13439 "MERGE_MSG",
13440 ] {
13441 let location = git::git(
13442 &path,
13443 &[
13444 "rev-parse",
13445 "--path-format=absolute",
13446 "--git-path",
13447 checkpoint,
13448 ],
13449 )
13450 .await?;
13451 anyhow::ensure!(
13452 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13453 if e.kind() == std::io::ErrorKind::NotFound),
13454 "Git recovery checkpoint retained: {checkpoint}"
13455 );
13456 }
13457 heads.insert(git::rev_parse(&path, "HEAD").await?);
13458 }
13459 crate::fold_evidence::preserve(state, home, &heads).await?;
13460
13461 Ok(cargo_guards)
13462}
13463
13464fn remove_if_empty(dir: &Path) {
13475 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13476 std::fs::remove_dir(dir).ok();
13477 }
13478}
13479
13480pub fn worst_open(state: &RunState) -> Option<Severity> {
13482 state
13483 .reviews
13484 .last()?
13485 .reviews
13486 .iter()
13487 .flat_map(|r| r.findings.iter())
13488 .map(|f| f.severity)
13489 .max()
13490}
13491
13492async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13496 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13497 git::acquire_worktree_config(repo).await?;
13498 state.enabled_worktree_config = true;
13499 }
13500 Ok(())
13501}
13502
13503#[cfg(test)]
13504mod tests {
13505 #[tokio::test]
13506 async fn launcher_base_fetch_requires_remote_success() {
13507 let tmp = tempfile::tempdir().unwrap();
13508 let repo = tmp.path().join("repo");
13509 let remote = tmp.path().join("remote");
13510 std::fs::create_dir(&repo).unwrap();
13511 std::fs::create_dir(&remote).unwrap();
13512 git::git(&remote, &["init", "--bare", "-b", "main"])
13513 .await
13514 .unwrap();
13515 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13516 git::git(
13517 &repo,
13518 &[
13519 "-c",
13520 "user.name=Fixture",
13521 "-c",
13522 "user.email=fixture@example.invalid",
13523 "commit",
13524 "--allow-empty",
13525 "-m",
13526 "fixture",
13527 ],
13528 )
13529 .await
13530 .unwrap();
13531 git::git(
13532 &repo,
13533 &["remote", "add", "origin", remote.to_str().unwrap()],
13534 )
13535 .await
13536 .unwrap();
13537 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13538 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13539 assert_eq!(
13540 resolve_base(&repo, "main", "origin", None)
13541 .await
13542 .unwrap()
13543 .commit,
13544 head
13545 );
13546 #[cfg(unix)]
13547 {
13548 let policy: crate::config::Coordination = toml::from_str(
13549 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13550 ).unwrap();
13551 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13552 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13553 let held = crate::repo_lock::SeatLease::operation_with(
13554 &repo,
13555 Some(&policy),
13556 &crate::repo_lock::WaitPolicy::default(),
13557 )
13558 .await
13559 .unwrap();
13560 assert!(held.is_some());
13561 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13562 tokio::pin!(fetch);
13563 assert!(
13564 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13565 .await
13566 .is_err()
13567 );
13568 assert!(
13569 !repo.join(".git/FETCH_HEAD").exists(),
13570 "no fetch before admission"
13571 );
13572 drop(held);
13573 let fetched = fetch.await.unwrap();
13574 assert_eq!(fetched.commit, head);
13575 assert_eq!(fetched.remote, "origin");
13576 assert_eq!(fetched.branch, "main");
13577 }
13578 git::git(
13580 &repo,
13581 &[
13582 "remote",
13583 "set-url",
13584 "origin",
13585 tmp.path().join("missing").to_str().unwrap(),
13586 ],
13587 )
13588 .await
13589 .unwrap();
13590 let error = resolve_base(&repo, "main", "origin", None)
13591 .await
13592 .unwrap_err()
13593 .to_string();
13594 assert!(error.contains("cannot read origin/main"));
13595 assert!(error.contains("refusing to branch off"));
13596 }
13597
13598 #[test]
13599 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13600 let leaked = vec!["chore(deps): update a crate".to_owned()];
13601 let own = "fix(daemon): apply a chosen action";
13602 assert!(should_retitle("", own, &leaked));
13603 assert!(should_retitle(
13604 &format!("{REVIEW_PROMPT_OPENING} `x`"),
13605 own,
13606 &leaked
13607 ));
13608 assert!(should_retitle(
13609 "chore: land candidate A of run 1",
13610 own,
13611 &leaked
13612 ));
13613 assert!(should_retitle(
13614 "magi: candidate A (uncommitted work)",
13615 own,
13616 &leaked
13617 ));
13618 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13619 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13620 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13621 }
13622
13623 #[test]
13624 fn pr_merge_plan_creates_adopts_or_stops() {
13625 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13626 assert_eq!(
13627 pr_merge_plan(Ok(land::OpenPr::One {
13628 url: "u".into(),
13629 title: "t".into()
13630 })),
13631 PrPlan::Adopt {
13632 url: "u".into(),
13633 title: "t".into()
13634 }
13635 );
13636 let PrPlan::Stop(many) =
13637 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13638 else {
13639 panic!("many must stop");
13640 };
13641 assert!(many.contains('a') && many.contains('b'));
13642 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13643 panic!("a failed lookup must stop");
13644 };
13645 assert!(err.contains("bad token"));
13646 }
13647
13648 use super::*;
13649 use crate::run::GateStatus;
13650 use std::collections::BTreeMap;
13651 use std::time::Duration;
13652
13653 fn conductor() -> AgentSpec {
13654 AgentSpec {
13655 id: "conductor".to_owned(),
13656 kind: crate::config::AgentKind::Command,
13657 model: None,
13658 command: vec!["true".to_owned()],
13659 extra_args: Vec::new(),
13660 env: BTreeMap::new(),
13661 prompt_delivery: None,
13662 max_turn_secs: None,
13663 }
13664 }
13665
13666 fn spec(id: &str) -> AgentSpec {
13667 AgentSpec {
13668 id: id.to_owned(),
13669 kind: crate::config::AgentKind::Command,
13670 model: None,
13671 command: vec!["true".to_owned()],
13672 extra_args: Vec::new(),
13673 env: BTreeMap::new(),
13674 prompt_delivery: None,
13675 max_turn_secs: None,
13676 }
13677 }
13678
13679 fn ids(xs: &[&str]) -> BTreeSet<String> {
13680 xs.iter().map(|s| (*s).to_owned()).collect()
13681 }
13682
13683 #[test]
13684 fn pick_successor_skips_an_agent_another_seat_holds() {
13685 let roster = [spec("a"), spec("b"), spec("c")];
13687 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13688 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13689 }
13690
13691 #[test]
13692 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13693 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13695 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13696 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13697 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13698 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13699 }
13700
13701 #[test]
13702 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13703 let roster = [spec("a"), spec("b")];
13704 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13705 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13706 let carried = ids(&["b"]);
13707 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13708 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13709 }
13710
13711 #[test]
13712 fn pick_successor_returns_none_without_any_untried_successor() {
13713 let roster = [spec("a"), spec("b")];
13714 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13715 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13716 let carried = ids(&["a"]);
13717 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13718 }
13719
13720 #[test]
13721 fn pick_successor_rescue_avoids_another_seats_occupant() {
13722 let roster = [spec("a"), spec("b"), spec("c")];
13724 let carried = ids(&["b", "c"]);
13725 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13726 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13727 }
13728
13729 #[test]
13730 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13731 let roster = [spec("a"), spec("b"), spec("c")];
13732 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13733 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13734 }
13735
13736 #[test]
13737 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13738 let roster = [spec("a"), spec("b"), spec("c")];
13739 let failed = ids(&["a", "b", "c"]);
13740 let mut tried = ids(&["b"]);
13742 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13743 assert_eq!(first.id, "a");
13744 tried.insert(first.id.clone());
13745 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13746 assert_eq!(second.id, "c");
13747 tried.insert(second.id.clone());
13748 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13749 }
13750
13751 #[test]
13752 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13753 let roster = [spec("a"), spec("b")];
13754 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13755 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13756 }
13757
13758 #[test]
13759 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13760 let roster = [spec("a"), spec("b"), spec("c")];
13761 let h = SeatHistory {
13762 failed: ids(&["a"]),
13763 last_ok: Some("b".to_owned()),
13764 last_fail: None,
13765 };
13766 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13767 let gone = SeatHistory {
13769 last_ok: Some("zzz".to_owned()),
13770 ..h.clone()
13771 };
13772 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13773 let failed = SeatHistory {
13774 failed: ids(&["a", "b"]),
13775 last_ok: Some("b".to_owned()),
13776 last_fail: None,
13777 };
13778 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13779 }
13780
13781 #[test]
13782 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13783 let first = SeatState::new("review-1", "alpha", 7);
13784 let next = handover_seat("review-1", "gamma", 7);
13785 assert_ne!(first.claude_session, next.claude_session);
13786 }
13787
13788 #[test]
13789 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13790 let mut state = state_with_summary("x", "y");
13791 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13792 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13793 assert_ne!(a.claude_session, b.claude_session);
13794 }
13795
13796 #[test]
13797 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13798 let roster = [spec("a"), spec("b")];
13799 let h = SeatHistory {
13800 failed: ids(&["a", "b"]),
13801 ..SeatHistory::default()
13802 };
13803 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13804 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13805 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13806 }
13807
13808 #[test]
13815 fn failure_signature_ignores_numbers_and_paths() {
13816 assert_eq!(
13817 failure_signature("exited with Some(2) and no usable output"),
13818 failure_signature("exited with Some(137) and no usable output")
13819 );
13820 assert_eq!(
13821 failure_signature("process exited with no usable output (exit code 2)"),
13822 failure_signature("process exited with no usable output (exit code 137)")
13823 );
13824 for (old, new) in [
13825 (
13826 "exited with Some(2) and no usable output",
13827 "process exited with no usable output (exit code 2)",
13828 ),
13829 (
13830 "exited with None and no usable output",
13831 "process exited with no usable output (exit code unavailable)",
13832 ),
13833 ] {
13834 let previous = FailClass::Other(failure_signature(old));
13835 let current = FailClass::Other(failure_signature(new));
13836 assert_eq!(previous, current, "resumption retains the failure class");
13837 assert!(!should_hand_over(Some(&previous), ¤t));
13838 }
13839 assert_eq!(
13840 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13841 failure_signature("cannot open /var/x.txt: denied")
13842 );
13843 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13844 }
13845
13846 #[test]
13847 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13848 let other = FailClass::Other("x".into());
13849 assert!(should_hand_over(None, &FailClass::Quota));
13850 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13851 assert!(should_hand_over(
13852 Some(&FailClass::Timeout),
13853 &FailClass::Timeout
13854 ));
13855 assert!(should_hand_over(None, &other));
13856 assert!(!should_hand_over(Some(&other), &other));
13857 assert!(should_hand_over(
13858 Some(&other),
13859 &FailClass::Other("y".into())
13860 ));
13861 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13863 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13864 }
13865
13866 #[test]
13867 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13868 let a = SeatState::new("judge-1", "alpha", 7);
13869 let b = handover_seat("judge-1", "beta", 7);
13870 assert_ne!(a.claude_session, b.claude_session);
13871 assert_eq!(b.turns, 0);
13872 }
13873
13874 #[test]
13875 fn a_timeout_is_classified_apart_from_other_failures() {
13876 assert_eq!(
13877 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13878 Some(FailClass::Timeout)
13879 );
13880 assert!(matches!(
13881 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13882 Some(FailClass::Other(_))
13883 ));
13884 }
13885
13886 #[test]
13887 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13888 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13889 let tried = BTreeSet::from(["beta".to_owned()]);
13890 let next = next_untried_in_roster(&roster, 1, &tried);
13893 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13894 }
13895
13896 #[test]
13897 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13898 let roster = vec![spec("alpha"), spec("beta")];
13899 let tried = BTreeSet::from(["beta".to_owned()]);
13900 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13904 }
13905
13906 #[test]
13907 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13908 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13909 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13910 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13914 }
13915
13916 #[test]
13917 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13918 let roster = vec![spec("a"), spec("a"), spec("b")];
13919 let tried = BTreeSet::from(["a".to_owned()]);
13920 let next = next_untried_in_roster(&roster, 0, &tried);
13921 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13922 }
13923
13924 #[test]
13925 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13926 let roster = vec![spec("a"), spec("b")];
13927 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13928 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13929 }
13930
13931 #[test]
13932 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13933 let dir = tempfile::tempdir().unwrap();
13934 let bay = dir.path().join("ffff");
13935
13936 remove_if_empty(&bay);
13938 assert!(!bay.exists());
13939
13940 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13943 remove_if_empty(&bay);
13944 assert!(bay.exists(), "non-empty directory must survive");
13945
13946 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13948 remove_if_empty(&bay);
13949 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13950 }
13951
13952 #[test]
13961 fn a_full_panel_that_found_nothing_is_clean() {
13962 assert!(round_is_clean(
13963 0,
13964 true,
13965 2,
13966 2,
13967 0,
13968 IncompleteReviewPolicy::Block
13969 ));
13970 }
13971
13972 #[test]
13973 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13974 assert!(!round_is_clean(
13975 0,
13976 true,
13977 1,
13978 2,
13979 0,
13980 IncompleteReviewPolicy::Block
13981 ));
13982 }
13983
13984 #[test]
13985 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13986 assert!(!round_is_clean(
13987 1,
13988 true,
13989 1,
13990 2,
13991 0,
13992 IncompleteReviewPolicy::Warn
13993 ));
13994 }
13995
13996 #[test]
13997 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13998 assert!(round_is_clean(
13999 0,
14000 true,
14001 1,
14002 2,
14003 0,
14004 IncompleteReviewPolicy::Warn
14005 ));
14006 }
14007
14008 #[test]
14009 fn a_full_panel_with_an_open_finding_is_not_clean() {
14010 assert!(!round_is_clean(
14011 1,
14012 true,
14013 2,
14014 2,
14015 0,
14016 IncompleteReviewPolicy::Block
14017 ));
14018 }
14019
14020 #[test]
14021 fn a_full_panel_with_a_red_e2e_is_not_clean() {
14022 assert!(!round_is_clean(
14023 0,
14024 false,
14025 2,
14026 2,
14027 0,
14028 IncompleteReviewPolicy::Block
14029 ));
14030 }
14031
14032 #[test]
14039 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14040 assert!(round_is_clean(
14043 0,
14044 true,
14045 1,
14046 2,
14047 1,
14048 IncompleteReviewPolicy::Block
14049 ));
14050 }
14051
14052 #[test]
14053 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14054 assert!(!round_is_clean(
14057 0,
14058 true,
14059 1,
14060 2,
14061 0,
14062 IncompleteReviewPolicy::Block
14063 ));
14064 }
14065
14066 #[test]
14067 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14068 assert!(!round_is_clean(
14069 1,
14070 true,
14071 1,
14072 2,
14073 1,
14074 IncompleteReviewPolicy::Block
14075 ));
14076 assert!(!round_is_clean(
14077 0,
14078 false,
14079 1,
14080 2,
14081 1,
14082 IncompleteReviewPolicy::Block
14083 ));
14084 }
14085
14086 #[test]
14087 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14088 assert!(!round_is_clean(
14092 0,
14093 true,
14094 0,
14095 2,
14096 2,
14097 IncompleteReviewPolicy::Block
14098 ));
14099 }
14100
14101 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14102 CommandOutcome {
14103 command: "test".to_owned(),
14104 code,
14105 output_tail: String::new(),
14106 duration_ms: 0,
14107 resource_blocked,
14108 }
14109 }
14110
14111 #[cfg(unix)]
14112 #[tokio::test]
14113 async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14114 ask_test_home();
14115 let temp = tempfile::tempdir().unwrap();
14116 let mut runner = runner_at(RunStatus::Gating);
14117 let shell = vec!["sh".to_owned(), "-c".to_owned()];
14118 let commands = vec![
14119 "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14120 "touch must-not-run".to_owned(),
14121 ];
14122 let (outcomes, retried, pids) = run_e2e_with_retry(
14123 &mut runner.state,
14124 &shell,
14125 &commands,
14126 temp.path(),
14127 Duration::from_secs(5),
14128 "fixture",
14129 )
14130 .await;
14131 assert_eq!(outcomes.len(), 1);
14132 assert!(outcomes[0].resource_blocked);
14133 assert!(!outcomes[0].build_failed());
14134 assert!(!retried);
14135 assert!(pids.is_empty());
14136 assert!(!temp.path().join("must-not-run").exists());
14137 assert_eq!(
14138 runner.state.resource_block.as_ref().unwrap().kind,
14139 "incremental cache admission"
14140 );
14141 }
14142
14143 #[test]
14144 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14145 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14146 assert!(
14147 !verify_inconclusive(&[outcome(Some(1), false)]),
14148 "an ordinary failure is still evidence about the patch"
14149 );
14150 assert!(verify_inconclusive(&[outcome(None, true)]));
14151 assert!(
14152 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14153 "one inconclusive outcome taints the whole batch"
14154 );
14155 assert!(!verify_inconclusive(&[]));
14156 }
14157
14158 #[tokio::test]
14159 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14160 let calls = std::sync::atomic::AtomicUsize::new(0);
14164 let started = Instant::now();
14165 wait_for_pids_with(
14166 &[123],
14167 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14168 Duration::from_millis(5),
14169 Duration::from_secs(5),
14170 )
14171 .await;
14172 assert!(
14173 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14174 "must keep checking rather than deciding on the first answer"
14175 );
14176 assert!(
14177 started.elapsed() < Duration::from_secs(1),
14178 "must return the moment it is confirmed dead, not wait out the ceiling"
14179 );
14180 }
14181
14182 #[tokio::test]
14183 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14184 let started = Instant::now();
14185 wait_for_pids_with(
14186 &[123],
14187 |_| true, Duration::from_millis(5),
14189 Duration::from_millis(30),
14190 )
14191 .await;
14192 let elapsed = started.elapsed();
14193 assert!(
14194 elapsed >= Duration::from_millis(30),
14195 "must not give up before its own ceiling: {elapsed:?}"
14196 );
14197 assert!(
14198 elapsed < Duration::from_secs(1),
14199 "must not wait past its own ceiling either: {elapsed:?}"
14200 );
14201 }
14202
14203 #[tokio::test]
14204 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14205 let started = Instant::now();
14206 wait_for_pids_with(
14207 &[],
14208 |_| true,
14209 Duration::from_secs(5),
14210 Duration::from_secs(5),
14211 )
14212 .await;
14213 assert!(
14214 started.elapsed() < Duration::from_millis(200),
14215 "an empty pid list has nothing to confirm"
14216 );
14217 }
14218
14219 fn review_round(
14225 clean: bool,
14226 blocking: usize,
14227 answered: usize,
14228 expected: usize,
14229 progressed: bool,
14230 e2e_ok: bool,
14231 ) -> ReviewRound {
14232 ReviewRound {
14233 round: 1,
14234 head: "h".to_owned(),
14235 verified_head: None,
14236 verified_at: None,
14237 reviews: Vec::new(),
14238 e2e: vec![CommandOutcome {
14239 command: "test".to_owned(),
14240 code: Some(if e2e_ok { 0 } else { 1 }),
14241 output_tail: String::new(),
14242 duration_ms: 0,
14243 resource_blocked: false,
14244 }],
14245 verify_retried: false,
14246 e2e_deferred: false,
14247 e2e_defer_reason: None,
14248 fix: None,
14249 blocking,
14250 answered,
14251 expected,
14252 clean,
14253 progressed,
14254 vote_split: false,
14255 reconsideration: Vec::new(),
14256 verdict: None,
14257 }
14258 }
14259
14260 #[test]
14261 fn review_conclusion_is_none_when_nothing_has_run() {
14262 assert_eq!(review_conclusion(&[], 3), None);
14263 }
14264
14265 #[test]
14266 fn review_conclusion_is_none_while_rounds_remain() {
14267 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14268 assert_eq!(review_conclusion(&rounds, 3), None);
14269 }
14270
14271 #[test]
14272 fn review_conclusion_is_gating_once_a_round_is_clean() {
14273 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14274 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14275 }
14276
14277 #[test]
14278 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14279 let rounds = vec![
14280 review_round(false, 1, 2, 2, true, true),
14281 review_round(false, 1, 2, 2, true, true),
14282 ];
14283 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14284 }
14285
14286 #[test]
14287 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14288 let rounds = vec![
14289 review_round(false, 1, 2, 2, true, true),
14290 review_round(false, 1, 2, 2, true, false),
14291 ];
14292 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14293 }
14294
14295 #[test]
14296 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14297 let mut blocked = review_round(false, 1, 2, 2, true, false);
14304 blocked.e2e[0].resource_blocked = true;
14305 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14306 assert_eq!(review_conclusion(&rounds, 2), None);
14307 }
14308
14309 #[test]
14310 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14311 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14313 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14314 }
14315
14316 #[test]
14317 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14318 let rounds = vec![
14319 review_round(false, 1, 2, 2, false, true),
14320 review_round(false, 1, 2, 2, false, true),
14321 ];
14322 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14323 }
14324
14325 fn secs(n: u64) -> Duration {
14326 Duration::from_secs(n)
14327 }
14328
14329 #[tokio::test]
14332 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14333 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14334 if std::env::var_os(CHILD).is_none() {
14335 let output = std::process::Command::new(std::env::current_exe().unwrap())
14339 .args([
14340 "--exact",
14341 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14342 "--nocapture",
14343 ])
14344 .env(CHILD, "1")
14345 .env_remove("MAGI_WORKER_HOME")
14346 .env_remove("MAGI_WORKER_ID")
14347 .output()
14348 .unwrap();
14349 assert!(
14350 output.status.success()
14351 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14352 "{}\n{}",
14353 String::from_utf8_lossy(&output.stdout),
14354 String::from_utf8_lossy(&output.stderr)
14355 );
14356 return;
14357 }
14358 ask_test_home();
14359 let tmp = tempfile::tempdir().unwrap();
14360 let repo = tmp.path().join("repo");
14361 std::fs::create_dir_all(&repo).unwrap();
14362 init_repo(&repo);
14363 let g = |args: &[&str]| {
14364 let out = std::process::Command::new("git")
14365 .args(args)
14366 .current_dir(&repo)
14367 .quiet()
14368 .output()
14369 .unwrap();
14370 assert!(
14371 out.status.success(),
14372 "{}",
14373 String::from_utf8_lossy(&out.stderr)
14374 );
14375 };
14376 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14377 g(&["checkout", "-b", "historical"]);
14378 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14379 g(&["add", "."]);
14380 g(&["commit", "-m", "old alternative"]);
14381 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14382 g(&["checkout", "-b", "magi/task-selected", &base]);
14383 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14384 g(&["add", "."]);
14385 g(&["commit", "-m", "measurement infrastructure"]);
14386 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14387 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14388 g(&["add", "."]);
14389 g(&["commit", "-m", "dependent implementation"]);
14390 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14391 g(&["checkout", "main"]);
14392 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14393 g(&["add", "."]);
14394 g(&["commit", "-m", "unrelated base change"]);
14395 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14396 let origin = tmp.path().join("origin.git");
14397 g(&[
14398 "clone",
14399 "--bare",
14400 repo.to_str().unwrap(),
14401 origin.to_str().unwrap(),
14402 ]);
14403 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14404 g(&["fetch", "origin"]);
14405 let selection = crate::pr_recovery::Selection {
14406 url: "https://example.test/project/pull/1".into(),
14407 branch: "magi/task-selected".into(),
14408 head: head.clone(),
14409 source_head: None,
14410 source_run: None,
14411 source_proof: None,
14412 disposition: Some(crate::pr_recovery::Disposition::Review),
14413 instruction: "preserve measured behavior".into(),
14414 approved_at: jiff::Timestamp::now(),
14415 };
14416 let mut template = runner_at(RunStatus::Reviewing);
14417 template.state.repo = repo.clone();
14418 template.state.base_commit = current.clone();
14419 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14420 template.state.origin = Some(crate::run::Origin::queue("selected"));
14421 template.state.pr_head = Some(selection.branch.clone());
14422 template.state.pr_recovery = Some(selection.clone());
14423 let mut runner =
14424 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14425 .await
14426 .unwrap();
14427 let wt = runner.state.winner().unwrap().worktree.clone();
14428 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14429 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14430 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14431 assert!(runner.state.seeds.is_empty());
14432 assert_eq!(runner.state.instruction, "preserve measured behavior");
14433 runner.sync_to_base().await.unwrap();
14434 assert_eq!(
14435 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14436 "prerequisite"
14437 );
14438 assert_eq!(
14439 std::fs::read_to_string(wt.join("implementation")).unwrap(),
14440 "measured implementation"
14441 );
14442 assert_eq!(
14443 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14444 "current main change"
14445 );
14446 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14447 assert_eq!(
14448 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14449 head
14450 );
14451
14452 let mut partial = runner_at(RunStatus::Reviewing);
14454 partial.state.repo = repo.clone();
14455 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14456 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14457 run: Some("exit 1".into()),
14458 ..Default::default()
14459 }];
14460 let selection = runner.state.pr_recovery.clone().unwrap();
14461 partial.state.pr_recovery = Some(selection.clone());
14462 partial.state.pr_head = Some(selection.branch.clone());
14463 partial.state.origin = Some(crate::run::Origin::queue("selected"));
14464 let id = partial.state.id.clone();
14465 let error = match Runner::open_selected(
14466 &repo,
14467 partial.state,
14468 partial.roles,
14469 1,
14470 runner.state.base_commit.clone(),
14471 selection,
14472 )
14473 .await
14474 {
14475 Ok(_) => panic!("setup must fail"),
14476 Err(e) => e,
14477 };
14478 assert_eq!(
14479 error
14480 .downcast_ref::<crate::run::StartupFailure>()
14481 .unwrap()
14482 .run,
14483 id
14484 );
14485 let failed = RunState::load(&id).unwrap();
14486 assert_eq!(failed.status, RunStatus::Failed);
14487 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14488 assert!(Runner::resume(&id).is_err());
14489 runner.state.status = RunStatus::Ready;
14490 runner.state.driver_exited = true;
14491 runner.state.save().unwrap();
14492 let mut retry = runner_at(RunStatus::Reviewing);
14493 retry.state.repo = repo.clone();
14494 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14495 let mut reselected = runner.state.pr_recovery.clone().unwrap();
14496 reselected.approved_at = jiff::Timestamp::now();
14497 retry.state.origin = Some(crate::run::Origin::queue("selected"));
14498 retry.state.pr_head = Some(reselected.branch.clone());
14499 retry.state.pr_recovery = Some(reselected.clone());
14500 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14501 .await
14502 .unwrap();
14503 let restarted = Runner::open_selected(
14504 &repo,
14505 retry.state,
14506 retry.roles,
14507 1,
14508 runner.state.base_commit.clone(),
14509 reselected,
14510 )
14511 .await
14512 .unwrap();
14513 assert_eq!(
14514 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14515 .await
14516 .unwrap(),
14517 head
14518 );
14519 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14520 assert_eq!(
14521 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14522 head
14523 );
14524 }
14525
14526 #[tokio::test]
14527 #[cfg(unix)]
14528 async fn fallback_and_interrupted_restoration_require_coordination() {
14529 use std::os::fd::AsRawFd as _;
14530 ask_test_home();
14531 let tmp = tempfile::tempdir().unwrap();
14532 let mut runner = registered_runner(tmp.path()).await;
14533 runner.state.config.graph.timeout_resource_wait = 1;
14534 let winner = runner.state.candidates[0].clone();
14535 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14536 let leftover = winner.worktree.join("leftover");
14537 std::fs::write(&leftover, "untrusted edits").unwrap();
14538 let next = tmp.path().join("next-fixer");
14539 let mut failed = spec("first");
14540 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14541 let mut successor = spec("second");
14542 successor.command = vec![
14543 "sh".into(),
14544 "-c".into(),
14545 format!("touch '{}'; echo done", next.display()),
14546 ];
14547 let attempts = vec![
14548 (failed.clone(), "first".into()),
14549 (successor.clone(), "second".into()),
14550 ];
14551 let lock_path = runner.state.repo.join(".git/updater.lock");
14552 let external = std::fs::File::create(&lock_path).unwrap();
14553 assert_eq!(
14554 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14555 0
14556 );
14557 let artifacts = tmp.path().join("artifacts");
14558 let build = |spec, seat| SeatJob {
14559 spec,
14560 seat,
14561 cwd: winner.worktree.clone(),
14562 prompt: "fixture".into(),
14563 timeout: Duration::from_secs(10),
14564 allow_write: true,
14565 sessions: false,
14566 artifacts: artifacts.clone(),
14567 stem: "fix".into(),
14568 handover: None,
14569 };
14570 let (_, _, out) = runner
14571 .ask_fixer_among(
14572 attempts.clone(),
14573 "graft",
14574 None,
14575 Some((&winner.worktree, &head)),
14576 build,
14577 )
14578 .await;
14579 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14580 assert!(leftover.exists(), "contention must prevent clean");
14581 assert!(!next.exists());
14582 assert!(runner.state.handovers.is_empty());
14583 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14585 from_run: "earlier".into(),
14586 generation: 1,
14587 mode: crate::recovery::Mode::VerifyRepair,
14588 reasons: Vec::new(),
14589 reviewed_head: None,
14590 rounds: Vec::new(),
14591 open_findings: Vec::new(),
14592 evidence: None,
14593 criteria_inherited: true,
14594 repair: crate::recovery::Repair::Started {
14595 before: head.clone(),
14596 },
14597 });
14598 assert!(
14599 runner
14600 .recovery_repair(&winner)
14601 .await
14602 .unwrap_err()
14603 .is::<crate::repo_lock::ResourceBlocked>()
14604 );
14605 assert!(matches!(
14606 runner.state.recovery.as_ref().unwrap().repair,
14607 crate::recovery::Repair::Started { .. }
14608 ));
14609 assert!(leftover.exists());
14610 drop(external);
14611 runner.recovery_repair(&winner).await.unwrap();
14612 assert!(!leftover.exists());
14613 assert!(matches!(
14614 runner.state.recovery.as_ref().unwrap().repair,
14615 crate::recovery::Repair::Done { .. }
14616 ));
14617 std::fs::remove_file(&lock_path).unwrap();
14619 std::fs::create_dir(&lock_path).unwrap();
14620 std::fs::write(&leftover, "untrusted edits").unwrap();
14621 let (_, _, out) = runner
14622 .ask_fixer_among(
14623 attempts,
14624 "graft",
14625 None,
14626 Some((&winner.worktree, &head)),
14627 build,
14628 )
14629 .await;
14630 assert!(matches!(out, AgentOutcome::Failed(_)));
14631 assert!(leftover.exists());
14632 assert!(!next.exists());
14633 assert_eq!(
14634 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14635 head
14636 );
14637 }
14638
14639 #[cfg(unix)]
14640 async fn registered_runner(root: &Path) -> Runner {
14641 repository_runner(root, crate::config::test_coordination()).await
14642 }
14643
14644 async fn repository_runner(
14645 root: &Path,
14646 coordination: Option<crate::config::Coordination>,
14647 ) -> Runner {
14648 ask_test_home();
14649 let repo = root.join("repo");
14650 std::fs::create_dir_all(&repo).unwrap();
14651 init_repo(&repo);
14652 if coordination.is_some() {
14653 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14654 }
14655 let mut runner = runner_at(RunStatus::Prep);
14656 runner.state.repo = repo.clone();
14657 runner.state.config.coordination = coordination;
14658 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14659 runner.state.config.graph.worktree_root = Some(root.join("wt"));
14660 runner.state.config.graph.advise = false;
14661 runner.roles.implementers = vec![spec("implement")];
14662 runner.roles.judges = vec![spec("judge")];
14663 runner.prep().await.unwrap();
14664 runner
14665 }
14666
14667 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14668
14669 #[tokio::test]
14670 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14671 let tmp = tempfile::tempdir().unwrap();
14672 let mut runner = repository_runner(tmp.path(), None).await;
14673 let prompts = runner.state.config.prompts.clone();
14674 let run = runner.state.id.clone();
14675 let ctx = WaveCtx {
14676 carry_seats: false,
14677 pauses: None,
14678 run: &run,
14679 node: "advise",
14680 prompts: &prompts,
14681 cache: None,
14682 round: None,
14683 };
14684 let jobs: Vec<_> = ["alpha", "beta"]
14685 .into_iter()
14686 .map(|id| SeatJob {
14687 spec: scripted(id, AUTH_OUTAGE),
14688 seat: SeatState::new(id, id, 7),
14689 prompt: "propose".into(),
14690 cwd: runner.state.candidates[0].worktree.clone(),
14691 timeout: Duration::from_secs(10),
14692 allow_write: false,
14693 sessions: false,
14694 artifacts: agent::artifacts_dir(&runner.state.dir()),
14695 stem: id.into(),
14696 handover: None,
14697 })
14698 .collect();
14699 let mut losses = Vec::new();
14700 let results = ask_wave_with(
14701 jobs.clone(),
14702 runner.sem.clone(),
14703 2,
14704 &[],
14705 &ctx,
14706 &mut losses,
14707 &mut runner.state,
14708 &|_: &str| Ok(()),
14709 )
14710 .await;
14711 assert!(
14712 results
14713 .iter()
14714 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14715 );
14716 assert_eq!(runner.state.quota.len(), 2);
14717 assert!(
14718 runner
14719 .state
14720 .quota
14721 .iter()
14722 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14723 );
14724 assert!(!runner.state.advise_attempted);
14725 runner.stop_for_resource().unwrap();
14726 assert_eq!(runner.state.status, RunStatus::Blocked);
14727 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14728 assert_eq!(
14729 runner.state.resource_block.as_ref().unwrap().kind,
14730 "agent authentication"
14731 );
14732 runner.state.resource_block = None;
14735 runner.state.quota.clear();
14736 let mut mixed = jobs;
14737 mixed[1].spec.command = vec!["false".into()];
14738 let results = ask_wave_with(
14739 mixed,
14740 runner.sem.clone(),
14741 0,
14742 &[],
14743 &ctx,
14744 &mut losses,
14745 &mut runner.state,
14746 &|_: &str| Ok(()),
14747 )
14748 .await;
14749 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14750 assert!(runner.state.resource_block.is_none());
14751 assert_eq!(losses.len(), 1);
14752 }
14753
14754 #[tokio::test]
14755 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14756 let tmp = tempfile::tempdir().unwrap();
14757 let mut runner = repository_runner(tmp.path(), None).await;
14758 let implementer = scripted("implement", AUTH_OUTAGE);
14759 runner.roles.implementers = vec![implementer.clone()];
14760 runner.state.config.agents = vec![implementer];
14761 runner.implement().await.unwrap();
14762 assert_eq!(runner.state.status, RunStatus::Blocked);
14763 assert_eq!(runner.state.quota.len(), 1);
14764 assert_eq!(
14765 runner.state.quota[0].cause,
14766 agent::QuotaCause::Authentication
14767 );
14768 let candidate = &runner.state.candidates[0];
14769 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14770 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14771 }
14772
14773 #[tokio::test]
14774 #[cfg(unix)]
14775 async fn parked_implementer_persists_completed_work() {
14776 for registered in [false, true] {
14777 let tmp = tempfile::tempdir().unwrap();
14778 let mut runner = registered_runner(tmp.path()).await;
14779 if !registered {
14780 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14781 }
14782 let marker = tmp.path().join("started");
14783 let release = tmp.path().join("release");
14784 let mut implementer = spec("implement");
14785 implementer.command = vec![
14786 "sh".into(),
14787 "-c".into(),
14788 format!(
14789 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14790 marker.display(),
14791 release.display()
14792 ),
14793 ];
14794 runner.roles.implementers = vec![implementer.clone()];
14795 runner.state.config.agents = vec![implementer];
14796 let pause = runner.pause.clone();
14797 let (result, ()) = tokio::join!(runner.implement(), async {
14798 tokio::time::timeout(Duration::from_secs(10), async {
14799 while !marker.exists() {
14800 tokio::time::sleep(Duration::from_millis(10)).await;
14801 }
14802 })
14803 .await
14804 .unwrap();
14805 pause.park();
14806 std::fs::write(&release, "go").unwrap();
14807 });
14808 result.unwrap();
14809 let candidate = &runner.state.candidates[0];
14810 assert!(candidate.commits > 0, "{candidate:?}");
14811 assert!(candidate.summary.contains("saved-result"));
14812 assert_eq!(candidate.files, 1);
14813 let before = candidate.commits;
14814 runner.implement().await.unwrap();
14815 assert_eq!(runner.state.candidates[0].commits, before);
14816 }
14817 }
14818
14819 #[tokio::test]
14820 #[cfg(unix)]
14821 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14822 let tmp = tempfile::tempdir().unwrap();
14823 let mut runner = registered_runner(tmp.path()).await;
14824 let winner = runner.state.candidates[0].clone();
14825 let cache = tmp.path().join("cache");
14826 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14827 runner.state.config.graph.timeout_resource_wait = 0;
14828 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14829 let crate::cache::AcquireOutcome::Acquired(held) =
14830 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14831 else {
14832 panic!("cache must be free")
14833 };
14834 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14835 from_run: "earlier".into(),
14836 generation: 1,
14837 mode: crate::recovery::Mode::VerifyRepair,
14838 reasons: Vec::new(),
14839 reviewed_head: None,
14840 rounds: Vec::new(),
14841 open_findings: Vec::new(),
14842 evidence: None,
14843 criteria_inherited: true,
14844 repair: crate::recovery::Repair::NotStarted,
14845 });
14846 runner.recovery_repair(&winner).await.unwrap();
14847 assert!(matches!(
14848 runner.state.recovery.as_ref().unwrap().repair,
14849 crate::recovery::Repair::WaitingForRestore { .. }
14850 ));
14851 drop(held);
14852 runner.state.resource_block = None;
14853 runner.state.config.graph.timeout_resource_wait = 1;
14854 runner.recovery_repair(&winner).await.unwrap();
14855 assert!(matches!(
14856 runner.state.recovery.as_ref().unwrap().repair,
14857 crate::recovery::Repair::Done { .. }
14858 ));
14859 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14860 }
14861
14862 #[tokio::test]
14863 #[cfg(unix)]
14864 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14865 use std::os::fd::AsRawFd as _;
14866 for graft in [false, true] {
14867 let tmp = tempfile::tempdir().unwrap();
14868 let mut runner = registered_runner(tmp.path()).await;
14869 runner.state.config.graph.timeout_resource_wait = 1;
14870 let winner = runner.state.candidates[0].clone();
14871 let marker = tmp.path().join("fixer-started");
14872 let mut fixer = spec("fixer");
14873 fixer.command = vec![
14874 "sh".into(),
14875 "-c".into(),
14876 format!("echo started >> '{}'; echo done", marker.display()),
14877 ];
14878 runner.roles.fixer = Some(vec![fixer]);
14879 if graft {
14880 let mut loser = winner.clone();
14881 loser.label = 'B';
14882 loser.index = 1;
14883 runner.state.candidates.push(loser);
14884 runner.state.config.graph.graft_runner_up = true;
14885 runner.state.tally = Some(Tally {
14886 first_choice: BTreeMap::new(),
14887 borda: BTreeMap::new(),
14888 winner: winner.label,
14889 rankings: 1,
14890 unanimous_initial: true,
14891 deliberated: false,
14892 changed_votes: 0,
14893 unanimous_final: true,
14894 tie_break: None,
14895 judges: 1,
14896 present: 1,
14897 quorum: 1,
14898 met_quorum: true,
14899 uncontested: None,
14900 });
14901 runner.state.judgements.push(crate::run::Judgement {
14902 judge: 1,
14903 seat: "judge-1".into(),
14904 agent: "judge".into(),
14905 ranking: vec![winner.label, 'B'],
14906 reasons: BTreeMap::new(),
14907 confidence: None,
14908 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14909 order: vec![0, 1],
14910 failed: None,
14911 duration_ms: 0,
14912 });
14913 } else {
14914 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14915 from_run: "earlier".into(),
14916 generation: 1,
14917 mode: crate::recovery::Mode::VerifyRepair,
14918 reasons: Vec::new(),
14919 reviewed_head: None,
14920 rounds: Vec::new(),
14921 open_findings: Vec::new(),
14922 evidence: None,
14923 criteria_inherited: true,
14924 repair: crate::recovery::Repair::NotStarted,
14925 });
14926 }
14927 let reader = crate::repo_lock::SeatLease::for_checkout(
14928 &winner.worktree,
14929 false,
14930 crate::config::test_coordination().as_ref(),
14931 )
14932 .await
14933 .unwrap();
14934 let pause = runner.pause.clone();
14935 let (result, ()) = tokio::join!(
14936 async {
14937 if graft {
14938 runner.graft().await
14939 } else {
14940 runner.recovery_repair(&winner).await
14941 }
14942 },
14943 async {
14944 tokio::time::sleep(Duration::from_millis(100)).await;
14945 pause.park();
14946 }
14947 );
14948 result.unwrap();
14949 assert!(!marker.exists(), "a blocked seat must not spawn");
14950 runner.state = RunState::load(&runner.state.id).unwrap();
14952 if graft {
14953 assert_eq!(
14954 runner.state.graft.as_ref().unwrap().outcome,
14955 GraftOutcome::WaitingForRestore
14956 );
14957 } else {
14958 assert!(matches!(
14959 runner.state.recovery.as_ref().unwrap().repair,
14960 crate::recovery::Repair::WaitingForRestore { .. }
14961 ));
14962 }
14963 drop(reader);
14964 runner.pause = Pause::new();
14965 runner.state.resource_block = None;
14966 let external =
14968 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14969 assert_eq!(
14970 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14971 0
14972 );
14973 let leftover = winner.worktree.join("leftover");
14974 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14975 let result = if graft {
14976 runner.graft().await
14977 } else {
14978 runner.recovery_repair(&winner).await
14979 };
14980 assert!(
14981 result
14982 .unwrap_err()
14983 .is::<crate::repo_lock::ResourceBlocked>()
14984 );
14985 assert!(leftover.exists());
14986 assert!(!marker.exists());
14987 drop(external);
14988 runner.state.resource_block = None;
14989 let result = if graft {
14990 runner.graft().await
14991 } else {
14992 runner.recovery_repair(&winner).await
14993 };
14994 result.unwrap();
14995 assert!(!leftover.exists(), "restore must precede retry");
14996 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14997 if graft {
14998 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14999 } else {
15000 assert!(matches!(
15001 runner.state.recovery.as_ref().unwrap().repair,
15002 crate::recovery::Repair::Done { .. }
15003 ));
15004 }
15005 }
15006 }
15007
15008 #[tokio::test]
15009 #[cfg(unix)]
15010 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15011 use std::os::fd::AsRawFd as _;
15012 let tmp = tempfile::tempdir().unwrap();
15013 let mut runner = registered_runner(tmp.path()).await;
15014 runner.state.config.graph.timeout_resource_wait = 1;
15015 let winner = runner.state.candidates[0].clone();
15016 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15017 runner.state.tally = Some(Tally {
15018 first_choice: BTreeMap::new(),
15019 borda: BTreeMap::new(),
15020 winner: winner.label,
15021 rankings: 0,
15022 unanimous_initial: false,
15023 deliberated: false,
15024 changed_votes: 0,
15025 unanimous_final: false,
15026 tie_break: None,
15027 judges: 0,
15028 present: 0,
15029 quorum: 0,
15030 met_quorum: true,
15031 uncontested: Some("fixture".into()),
15032 });
15033 runner.state.graft = Some(GraftRecord {
15034 pre: pre.clone(),
15035 post: None,
15036 lines: None,
15037 limit: None,
15038 agent: None,
15039 outcome: GraftOutcome::Running,
15040 requested: Vec::new(),
15041 adoption: None,
15042 adoption_error: None,
15043 });
15044 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15045 git::commit_all(&winner.worktree, "unfinished graft")
15046 .await
15047 .unwrap();
15048 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15049 let leftover = winner.worktree.join("leftover");
15050 std::fs::write(&leftover, "untrusted edits").unwrap();
15051 let path = runner.state.repo.join(".git/updater.lock");
15052 let external = std::fs::File::create(&path).unwrap();
15053 assert_eq!(
15054 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15055 0
15056 );
15057 assert!(
15058 runner
15059 .graft()
15060 .await
15061 .unwrap_err()
15062 .is::<crate::repo_lock::ResourceBlocked>()
15063 );
15064 assert!(matches!(
15065 runner.state.graft.as_ref().unwrap().outcome,
15066 GraftOutcome::Running
15067 ));
15068 assert_eq!(
15069 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15070 unfinished
15071 );
15072 assert!(leftover.exists());
15073 drop(external);
15074 std::fs::remove_file(&path).unwrap();
15075 std::fs::create_dir(&path).unwrap();
15076 assert!(runner.graft().await.is_err());
15077 assert!(matches!(
15078 runner.state.graft.as_ref().unwrap().outcome,
15079 GraftOutcome::Running
15080 ));
15081 assert_eq!(
15082 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15083 unfinished
15084 );
15085 assert!(leftover.exists());
15086 std::fs::remove_dir(&path).unwrap();
15087 runner.graft().await.unwrap();
15088 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15089 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15090 assert!(!leftover.exists());
15091 }
15092
15093 #[tokio::test]
15094 #[cfg(unix)]
15095 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15096 use std::os::fd::AsRawFd as _;
15097 let tmp = tempfile::tempdir().unwrap();
15098 let mut runner = registered_runner(tmp.path()).await;
15099 runner.state.prep_complete = Some(false);
15100 runner.state.config.graph.timeout_resource_wait = 1;
15101 runner.state.save().unwrap();
15102 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15103 assert_eq!(
15104 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15105 0
15106 );
15107 runner.execute().await.unwrap();
15108 let disk = RunState::load(&runner.state.id).unwrap();
15109 assert!(disk.driver_exited);
15110 assert_eq!(disk.driver_pid, None);
15111 assert_eq!(runner.state.status, RunStatus::Blocked);
15112 assert_eq!(
15113 runner.state.resource_block.as_ref().unwrap().kind,
15114 "repository coordination"
15115 );
15116 assert!(runner.state.prep_failure.is_none());
15117 assert!(runner.state.seats.is_empty());
15118 runner.pause.park();
15119 tokio::time::timeout(Duration::from_secs(10), runner.execute())
15122 .await
15123 .unwrap()
15124 .unwrap();
15125 assert!(runner.state.parked);
15126 assert!(runner.state.prep_failure.is_none());
15127 let disk = RunState::load(&runner.state.id).unwrap();
15128 assert!(disk.driver_exited);
15129 assert_eq!(disk.driver_pid, None);
15130 }
15131
15132 #[tokio::test]
15133 #[cfg(unix)]
15134 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15135 use std::os::fd::AsRawFd as _;
15136 let tmp = tempfile::tempdir().unwrap();
15137 let mut runner = registered_runner(tmp.path()).await;
15138 runner.state.prep_complete = Some(false);
15139 runner.state.config.graph.timeout_resource_wait = 30;
15140 runner.state.save().unwrap();
15141 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15142 assert_eq!(
15143 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15144 0
15145 );
15146 assert!(
15147 tokio::time::timeout(Duration::from_millis(100), runner.execute())
15148 .await
15149 .is_err()
15150 );
15151 let disk = RunState::load(&runner.state.id).unwrap();
15152 assert!(disk.driver_exited);
15153 assert_eq!(disk.driver_pid, None);
15154 assert!(disk.status.resumable());
15155 assert!(runner.state.driver_exited);
15156 assert_eq!(runner.state.driver_pid, None);
15157 runner.state.save().unwrap();
15158 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15159 }
15160
15161 #[tokio::test]
15162 #[cfg(unix)]
15163 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15164 let tmp = tempfile::tempdir().unwrap();
15165 let mut runner = registered_runner(tmp.path()).await;
15166 let winner = runner.state.candidates[0].clone();
15167 let cache = tmp.path().join("cache");
15168 let started = tmp.path().join("pre-gate-started");
15169 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15170 runner.state.config.verify.gate =
15171 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15172 let home = crate::run::home();
15173 let owner =
15174 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15175 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15176 crate::cache::AcquireOutcome::Acquired(g) => g,
15177 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15178 };
15179 let mut gate = Box::pin(runner.run_gate(&winner));
15180 assert!(
15181 tokio::time::timeout(Duration::from_millis(150), &mut gate)
15182 .await
15183 .is_err()
15184 );
15185 assert!(!started.exists(), "pre-gate must wait for the cache too");
15186 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15190 let editor = crate::repo_lock::SeatLease::checkout_with(
15191 &winner.worktree,
15192 true,
15193 crate::config::test_coordination().as_ref(),
15194 &wait,
15195 )
15196 .await
15197 .unwrap();
15198 drop(editor);
15199 guard.release();
15200 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15201 .await
15202 .unwrap()
15203 .unwrap();
15204 drop(gate);
15205 assert!(outcomes.iter().all(CommandOutcome::ok));
15206 assert!(started.exists());
15207 }
15208
15209 #[tokio::test]
15210 #[cfg(unix)]
15211 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15212 use std::os::fd::AsRawFd as _;
15213
15214 ask_test_home();
15215 let tmp = tempfile::tempdir().unwrap();
15216 let repo = tmp.path().join("repo");
15217 std::fs::create_dir_all(&repo).unwrap();
15218 init_repo(&repo);
15219 let common = repo.join(".git");
15220 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15221 let mut runner = runner_at(RunStatus::Prep);
15222 runner.state.repo = repo.clone();
15223 runner.state.config.coordination = crate::config::test_coordination();
15224 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15225 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15226 runner.state.config.graph.advise = false;
15227 runner.roles.implementers = vec![spec("implement")];
15228 runner.roles.judges = vec![spec("judge")];
15229 runner.prep().await.unwrap();
15230 let winner = runner.state.candidates[0].clone();
15231 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15234 let completed = tmp.path().join("seat-completed");
15235 runner.roles.implementers[0].command = vec![
15236 "sh".into(),
15237 "-c".into(),
15238 format!("touch '{}'; echo completed", completed.display()),
15239 ];
15240 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15241 assert_eq!(
15242 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15243 0
15244 );
15245 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15246 let mut implement = Box::pin(runner.implement());
15247 tokio::select! {
15248 result = &mut implement => panic!("rescue must wait: {result:?}"),
15249 result = tokio::time::timeout(Duration::from_secs(10), async {
15250 while !completed.exists() {
15251 tokio::time::sleep(Duration::from_millis(20)).await;
15252 }
15253 }) => result.unwrap(),
15254 }
15255 assert!(
15256 tokio::time::timeout(Duration::from_millis(200), &mut implement)
15257 .await
15258 .is_err()
15259 );
15260 assert_eq!(
15261 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15262 before
15263 );
15264 assert_eq!(
15265 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15266 0
15267 );
15268 tokio::time::timeout(Duration::from_secs(10), &mut implement)
15269 .await
15270 .unwrap()
15271 .unwrap();
15272 drop(implement);
15273 assert_ne!(
15274 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15275 before
15276 );
15277 assert!(git::is_clean(&winner.worktree).await.unwrap());
15278
15279 let started = tmp.path().join("pre-gate-started");
15280 let proceed = tmp.path().join("pre-gate-proceed");
15281 let checked = tmp.path().join("gate-ran");
15282 runner.state.config.verify.pre_gate = vec![format!(
15283 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15284 started.display(),
15285 proceed.display()
15286 )];
15287 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15288 assert_eq!(
15289 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15290 0
15291 );
15292 let mut gate = Box::pin(runner.run_gate(&winner));
15293 assert!(
15294 tokio::time::timeout(Duration::from_millis(200), &mut gate)
15295 .await
15296 .is_err()
15297 );
15298 assert!(!started.exists());
15299 assert!(!checked.exists());
15300 assert!(git::is_clean(&winner.worktree).await.unwrap());
15301 assert_eq!(
15302 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15303 0
15304 );
15305 tokio::select! {
15306 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15307 result = tokio::time::timeout(Duration::from_secs(10), async {
15308 while !started.exists() {
15309 tokio::time::sleep(Duration::from_millis(20)).await;
15310 }
15311 }) => result.unwrap(),
15312 }
15313 assert_eq!(
15314 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15315 -1,
15316 "mechanical edits must hold the real lock"
15317 );
15318 std::fs::write(proceed, "ready").unwrap();
15319 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15320 .await
15321 .unwrap()
15322 .unwrap();
15323 drop(gate);
15324 assert!(outcomes.iter().all(CommandOutcome::ok));
15325 assert!(checked.exists());
15326 assert!(runner.state.pre_gate_commit.is_some());
15327 assert!(git::is_clean(&winner.worktree).await.unwrap());
15328 assert_eq!(
15329 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15330 0,
15331 "gate completion must release coordination"
15332 );
15333 drop(outsider);
15334 std::fs::remove_file(common.join("updater.lock")).unwrap();
15335 std::fs::create_dir(common.join("updater.lock")).unwrap();
15336 std::fs::remove_file(&started).unwrap();
15337 std::fs::remove_file(&checked).unwrap();
15338 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15339 let error = runner.run_gate(&winner).await.unwrap_err();
15340 assert!(
15341 error
15342 .to_string()
15343 .contains("acquire mandatory repository lock")
15344 );
15345 assert!(
15346 !started.exists(),
15347 "failed acquisition must prevent pre-gate"
15348 );
15349 assert!(!checked.exists(), "failed acquisition must prevent gate");
15350 assert!(git::is_clean(&winner.worktree).await.unwrap());
15351 assert_eq!(
15352 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15353 head
15354 );
15355 }
15356
15357 #[tokio::test]
15358 #[cfg(unix)]
15359 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15360 use std::os::fd::AsRawFd as _;
15361 use std::time::Duration;
15362
15363 ask_test_home();
15364 let tmp = tempfile::tempdir().unwrap();
15365 let repo = tmp.path().join("repo");
15366 std::fs::create_dir_all(&repo).unwrap();
15367 init_repo(&repo);
15368 let common = repo.join(".git");
15369 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15370 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15371 assert_eq!(
15372 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15373 0
15374 );
15375 let mut runner = runner_at(RunStatus::Prep);
15376 runner.state.repo = repo.clone();
15377 runner.state.config.coordination = crate::config::test_coordination();
15378 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15379 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15380 runner.state.config.graph.advise = true;
15381 runner.state.config.graph.advisors = 1;
15382 runner.roles.implementers = vec![spec("implement")];
15383 runner.roles.judges = vec![spec("judge")];
15384 let root = runner.state.worktree_root();
15385 let started = tmp.path().join("setup-started");
15386 let proceed = tmp.path().join("setup-proceed");
15387 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15388 run: Some(format!(
15389 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15390 started.display(),
15391 proceed.display()
15392 )),
15393 ..Default::default()
15394 }];
15395 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15396 let config = git::git(&repo, &["config", "--local", "--list"])
15397 .await
15398 .unwrap();
15399 let mut prep = Box::pin(runner.prep());
15400 assert!(
15401 tokio::time::timeout(Duration::from_millis(200), &mut prep)
15402 .await
15403 .is_err()
15404 );
15405 assert!(
15406 !root.exists(),
15407 "contention must prevent even checkout creation"
15408 );
15409 assert!(!started.exists(), "contention must prevent setup");
15410 assert_eq!(
15411 git::git(&repo, &["branch", "--list"]).await.unwrap(),
15412 branches
15413 );
15414 assert_eq!(
15415 git::git(&repo, &["config", "--local", "--list"])
15416 .await
15417 .unwrap(),
15418 config
15419 );
15420 drop(lock);
15421 tokio::select! {
15422 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15423 result = tokio::time::timeout(Duration::from_secs(10), async {
15424 while !started.exists() {
15425 tokio::time::sleep(Duration::from_millis(20)).await;
15426 }
15427 }) => result.unwrap(),
15428 }
15429 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15430 assert_eq!(
15431 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15432 -1
15433 );
15434 assert!(
15435 tokio::time::timeout(
15436 Duration::from_millis(100),
15437 crate::repo_lock::SeatLease::for_checkout(
15438 &repo,
15439 true,
15440 crate::config::test_coordination().as_ref()
15441 )
15442 )
15443 .await
15444 .is_err(),
15445 "preparation must also exclude local editors"
15446 );
15447 std::fs::write(proceed, "ready").unwrap();
15448 tokio::time::timeout(Duration::from_secs(10), &mut prep)
15449 .await
15450 .unwrap()
15451 .unwrap();
15452 drop(prep);
15453 assert_eq!(runner.state.prep_complete, Some(true));
15454 assert!(root.join("judge-1").exists());
15455 assert!(root.join("advisor-1").exists());
15456 assert_eq!(
15457 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15458 0
15459 );
15460 }
15461
15462 #[tokio::test]
15463 #[cfg(unix)]
15464 async fn preparation_lock_failure_prevents_repository_changes() {
15465 ask_test_home();
15466 let tmp = tempfile::tempdir().unwrap();
15467 let repo = tmp.path().join("repo");
15468 std::fs::create_dir_all(&repo).unwrap();
15469 init_repo(&repo);
15470 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15471 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15474 let mut runner = runner_at(RunStatus::Prep);
15475 runner.state.repo = repo.clone();
15476 runner.state.config.coordination = crate::config::test_coordination();
15477 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15478 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15479 runner.roles.implementers = vec![spec("implement")];
15480 let error = runner.prep().await.unwrap_err();
15481 assert!(
15482 error
15483 .to_string()
15484 .contains("acquire mandatory repository lock")
15485 );
15486 assert!(!runner.state.worktree_root().exists());
15487 assert!(runner.state.candidates.is_empty());
15488 assert!(!repo.join(".git/config.worktree").exists());
15489 assert_eq!(
15490 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15491 "* main"
15492 );
15493 }
15494
15495 #[tokio::test]
15496 async fn preparation_records_creation_intent_before_git_can_fail() {
15497 ask_test_home();
15498 let tmp = tempfile::tempdir().unwrap();
15499 let repo = tmp.path().join("repo");
15500 std::fs::create_dir_all(&repo).unwrap();
15501 init_repo(&repo);
15502 let mut runner = runner_at(RunStatus::Prep);
15503 runner.state.repo = repo.clone();
15504 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15505 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15506 runner.state.config.blind.commit_msg_hook = false;
15507 runner.roles.implementers = vec![spec("one")];
15508 let label = blind::assign_labels(1, runner.state.seed)[0];
15509 let branch = runner.state.branch_for(label);
15510 git::git(&repo, &["branch", &branch]).await.unwrap();
15513 assert!(runner.prep().await.is_err());
15514 let saved = RunState::load(&runner.state.id).unwrap();
15515 assert!(saved.candidates.is_empty());
15516 let event = saved
15517 .events
15518 .iter()
15519 .find(|e| e.node == "prep-intent")
15520 .unwrap();
15521 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15522 assert_eq!(intent["branch"], branch);
15523 assert_eq!(intent["start"], runner.state.base_commit);
15524 assert_eq!(
15525 intent["worktree"],
15526 runner
15527 .state
15528 .worktree_root()
15529 .join(format!("cand-{label}"))
15530 .to_string_lossy()
15531 .as_ref()
15532 );
15533 assert_eq!(intent["picks"], serde_json::json!([]));
15534 }
15535
15536 #[tokio::test]
15537 async fn invalid_pinned_sources_create_no_partial_worktrees() {
15538 ask_test_home();
15539 let tmp = tempfile::tempdir().unwrap();
15540 let repo = tmp.path().join("repo");
15541 std::fs::create_dir_all(&repo).unwrap();
15542 init_repo(&repo);
15543 let mut runner = runner_at(RunStatus::Prep);
15544 runner.state.repo = repo.clone();
15545 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15546 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15547 runner.state.sources.pick_commits.push(refs::Input {
15548 token: "missing".into(),
15549 sha: "0".repeat(40),
15550 provenance: refs::Provenance::ExplicitCli,
15551 });
15552 assert!(runner.prep().await.is_err());
15553 assert!(runner.state.candidates.is_empty());
15554 assert!(!runner.state.worktree_root().exists());
15555 }
15556
15557 #[tokio::test]
15558 async fn prose_and_legacy_inference_never_import_into_candidates() {
15559 ask_test_home();
15560 let tmp = tempfile::tempdir().unwrap();
15561 let repo = tmp.path().join("repo");
15562 std::fs::create_dir_all(&repo).unwrap();
15563 init_repo(&repo);
15564 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15565 git::git(&repo, &["checkout", "-b", "magi/history/A"])
15566 .await
15567 .unwrap();
15568 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15569 git::git(&repo, &["add", "."]).await.unwrap();
15570 git::git(&repo, &["commit", "-m", "historical implementation"])
15571 .await
15572 .unwrap();
15573 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15574 git::git(&repo, &["checkout", "main"]).await.unwrap();
15575 for instruction in [
15576 format!("Example: magi/history/A at {source}"),
15577 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15578 format!("Never import magi/history/A or {source}"),
15579 format!("Conversation answer: I mentioned {source} only as background"),
15580 ] {
15581 let mut runner = runner_at(RunStatus::Prep);
15582 runner.state.repo = repo.clone();
15583 runner.state.base_commit = base.clone();
15584 runner.state.instruction = instruction.clone();
15585 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15586 runner.state.config.blind.commit_msg_hook = false;
15587 runner.state.seeds = vec![refs::Seed {
15588 token: "magi/history/A".into(),
15589 sha: source.clone(),
15590 kind: refs::SeedKind::Unmerged,
15591 branch: true,
15592 detail: String::new(),
15593 }];
15594 runner.roles.implementers = vec![spec("one")];
15595 assert_eq!(seeded_instruction(&runner.state), instruction);
15596 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15598 legacy.as_object_mut().unwrap().remove("sources");
15599 runner.state = serde_json::from_value(legacy).unwrap();
15600 assert_eq!(runner.state.sources, refs::Sources::default());
15601 runner.state.save().unwrap();
15602 runner.state = RunState::load(&runner.state.id).unwrap();
15603 runner.prep().await.unwrap();
15604 for candidate in &runner.state.candidates {
15605 assert_eq!(
15606 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15607 base
15608 );
15609 assert!(!candidate.worktree.join("unwanted").exists());
15610 }
15611 }
15612 }
15613
15614 #[tokio::test]
15615 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15616 ask_test_home();
15617 let tmp = tempfile::tempdir().unwrap();
15618 let repo = tmp.path().join("repo");
15619 std::fs::create_dir_all(&repo).unwrap();
15620 init_repo(&repo);
15621 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15622 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15623 std::fs::write(repo.join("referenced"), "required commit").unwrap();
15624 git::git(&repo, &["add", "."]).await.unwrap();
15625 git::git(&repo, &["commit", "-m", "required seed"])
15626 .await
15627 .unwrap();
15628 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15629 git::git(&repo, &["checkout", "main"]).await.unwrap();
15630 let mut runner = runner_at(RunStatus::Prep);
15631 runner.state.repo = repo.clone();
15632 runner.state.base_commit = base.clone();
15633 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15634 runner.state.config.graph.advise = false;
15635 runner.state.config.blind.commit_msg_hook = false;
15636 runner.state.sources =
15637 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15638 .await
15639 .unwrap();
15640 runner.roles.implementers = vec![spec("one"), spec("two")];
15641 runner.roles.judges = vec![spec("judge")];
15642 let label = blind::assign_labels(2, runner.state.seed)[0];
15643 let branch = runner.state.branch_for(label);
15644 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15645 git::worktree_add_branch(&repo, &worktree, &branch, &base)
15646 .await
15647 .unwrap();
15648 runner.state.candidates.push(Candidate {
15649 index: 0,
15650 label,
15651 agent: "one".into(),
15652 branch,
15653 worktree,
15654 summary: String::new(),
15655 stat: String::new(),
15656 files: 0,
15657 commits: 0,
15658 empty: false,
15659 failed: None,
15660 verified_noop: None,
15661 duration_ms: 0,
15662 folded: false,
15663 });
15664 runner.state.save().unwrap();
15665 runner.state = RunState::load(&runner.state.id).unwrap();
15667 assert_eq!(runner.state.prep_complete, Some(false));
15668 runner.prep().await.unwrap();
15669 assert_eq!(runner.state.prep_complete, Some(true));
15670 assert_eq!(runner.state.candidates.len(), 2);
15671 for candidate in &runner.state.candidates {
15672 assert_eq!(
15673 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15674 git::tree_of(&repo, &seed).await.unwrap()
15675 );
15676 assert_eq!(
15677 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15678 "required commit"
15679 );
15680 }
15681 assert!(runner.state.worktree_root().join("judge-1").exists());
15682 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15683 runner.prep().await.unwrap();
15684 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15685
15686 runner.state.prep_complete = Some(false);
15687 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15688 run: Some(format!(
15689 "test -f '{}'",
15690 tmp.path().join("dependency-ready").display()
15691 )),
15692 ..Default::default()
15693 }];
15694 assert!(runner.prep().await.is_err());
15695 assert!(runner.state.prep_failure.is_some());
15696 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15697 runner.state = RunState::load(&runner.state.id).unwrap();
15698 assert!(
15699 runner.prep().await.is_err(),
15700 "correcting the cause alone must not auto-retry"
15701 );
15702 let failed_at = runner.state.events.last().unwrap().at;
15703 assert!(
15704 runner.retry_preparation(failed_at).is_err(),
15705 "old approvals cannot unlock the hold"
15706 );
15707 let mut task = crate::queue::Task::new(
15708 "retry".into(),
15709 "retry setup".into(),
15710 repo.clone(),
15711 crate::queue::Source::Human,
15712 );
15713 task.start(runner.state.id.clone());
15714 task.hold_machine(Some("setup failed".into()));
15715 task.release_operator("cli", None);
15716 runner
15717 .retry_preparation(task.resumption.as_ref().unwrap().at)
15718 .unwrap();
15719 runner.prep().await.unwrap();
15720 assert!(runner.state.prep_failure.is_none());
15721 assert_eq!(runner.state.prep_complete, Some(true));
15722 assert_eq!(runner.state.candidates.len(), 2);
15723 assert!(runner.state.worktree_root().join("judge-1").exists());
15724 for candidate in &runner.state.candidates {
15725 assert_eq!(
15726 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15727 git::tree_of(&repo, &seed).await.unwrap()
15728 );
15729 }
15730 }
15731
15732 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15733 let mut heads = Vec::new();
15734 for candidate in candidates {
15735 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15736 }
15737 heads
15738 }
15739
15740 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15741 ask_test_home();
15742 let tmp = tempfile::tempdir().unwrap();
15743 let repo = tmp.path().join("repo");
15744 std::fs::create_dir(&repo).unwrap();
15745 init_repo(&repo);
15746 let mut runner = runner_at(RunStatus::Reviewing);
15747 runner.state.repo = repo.clone();
15748 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15749 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15750 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15751 let branch = format!("candidate-{label}");
15752 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15753 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15754 .await
15755 .unwrap();
15756 if label == 'B' {
15757 std::fs::write(worktree.join("change"), "winner").unwrap();
15758 git::rescue_commit(&worktree, "candidate change")
15759 .await
15760 .unwrap();
15761 }
15762 runner.state.candidates.push(Candidate {
15763 index,
15764 label,
15765 agent: "alpha".into(),
15766 branch,
15767 worktree,
15768 summary: "completed reply".into(),
15769 stat: String::new(),
15770 files: usize::from(label == 'B'),
15771 commits: usize::from(label == 'B'),
15772 empty: label == 'A',
15773 failed: None,
15774 verified_noop: None,
15775 duration_ms: 1,
15776 folded: false,
15777 });
15778 }
15779 runner.state.judge_skipped = true;
15780 runner.state.tally = Some(Tally {
15781 first_choice: BTreeMap::new(),
15782 borda: BTreeMap::new(),
15783 winner: 'B',
15784 rankings: 0,
15785 unanimous_initial: false,
15786 deliberated: false,
15787 changed_votes: 0,
15788 unanimous_final: false,
15789 tie_break: None,
15790 judges: 0,
15791 present: 0,
15792 quorum: 0,
15793 met_quorum: true,
15794 uncontested: Some("only B produced a change".into()),
15795 });
15796 runner.state.save().unwrap();
15797 (tmp, runner)
15798 }
15799
15800 #[tokio::test]
15801 #[cfg(unix)]
15802 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15803 for (solo, failed_checkpoint) in
15804 [(true, false), (false, false), (true, true), (false, true)]
15805 {
15806 let (_tmp, mut runner) = preservation_resume_fixture().await;
15807 if solo {
15808 runner.state.candidates.retain(|c| c.label == 'A');
15809 }
15810 runner.state.candidates[0].empty = false;
15811 runner.state.candidates[0].summary.clear();
15812 runner.state.tally = None;
15813 runner.state.judge_skipped = false;
15814 runner.state.status = RunStatus::Implementing;
15815 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15816 let mut implementer = spec("alpha");
15817 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15818 runner.roles.implementers = vec![implementer];
15819 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15820 if failed_checkpoint {
15821 std::fs::create_dir_all(&patch).unwrap();
15822 }
15823 runner.state.save().unwrap();
15824 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15825 drop(pressure);
15826 runner.state = RunState::load(&runner.state.id).unwrap();
15827 assert_eq!(runner.state.status, RunStatus::Blocked);
15828 assert!(runner.state.interrupted_implementations.contains(&'A'));
15829 assert_eq!(runner.state.candidates[0].commits, 0);
15830 assert!(!runner.state.candidates[0].empty);
15831 assert!(runner.state.candidates[0].failed.is_none());
15832 if failed_checkpoint {
15833 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15834 std::fs::remove_dir(patch).unwrap();
15835 }
15836 runner.state.resource_block = None;
15837 runner.state.config.disk.min_free_bytes = 0;
15838 runner.roles.implementers[0].command = vec![
15839 "sh".into(),
15840 "-c".into(),
15841 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15842 ];
15843 runner.implement().await.unwrap();
15844 assert!(runner.state.interrupted_implementations.is_empty());
15845 assert!(runner.state.candidate_preservation.is_empty());
15846 assert_eq!(runner.state.candidates[0].commits, 1);
15847 assert!(!runner.state.candidates[0].empty);
15848 assert!(
15849 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15850 .unwrap()
15851 .contains("retried empty interrupted seat")
15852 );
15853 if !solo {
15854 assert_eq!(runner.state.candidates[1].commits, 1);
15855 }
15856 }
15857 }
15858
15859 #[tokio::test]
15860 #[cfg(unix)]
15861 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15862 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15863 let (_tmp, mut runner) = preservation_resume_fixture().await;
15864 runner.state.candidates.retain(|c| c.label == 'A');
15865 runner.state.candidates[0].empty = false;
15866 runner.state.candidates[0].summary.clear();
15867 runner.state.tally = None;
15868 runner.state.judge_skipped = false;
15869 runner.state.status = RunStatus::Implementing;
15870 let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15873 let wt = runner.state.candidates[0].worktree.clone();
15874 std::fs::write(
15875 wt.join("interrupted-source"),
15876 "retain implementation work\n",
15877 )
15878 .unwrap();
15879 if committed {
15880 git::rescue_commit(&wt, "implementation checkpoint")
15881 .await
15882 .unwrap();
15883 }
15884 let lock = git::git(
15885 &wt,
15886 &[
15887 "rev-parse",
15888 "--path-format=absolute",
15889 "--git-path",
15890 "index.lock",
15891 ],
15892 )
15893 .await
15894 .unwrap();
15895 if block_rescue {
15896 std::fs::write(&lock, "fixture-owned lock").unwrap();
15897 }
15898 let mut implementer = spec("alpha");
15899 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15900 runner.roles.implementers = vec![implementer];
15901 runner.state.save().unwrap();
15902 let result = runner.implement().await;
15903 drop(pressure);
15904 assert_eq!(result.is_err(), block_rescue);
15905 assert_eq!(
15906 runner.state.resource_block.as_ref().unwrap().kind,
15907 "disk pressure"
15908 );
15909 assert!(
15910 runner
15911 .state
15912 .events
15913 .iter()
15914 .any(|e| e.message.contains("agent execution interrupted"))
15915 );
15916 assert!(
15917 runner
15918 .state
15919 .dir()
15920 .join("artifacts/candidate-A-recovery/staged.patch")
15921 .exists()
15922 );
15923 if block_rescue {
15924 let saved = RunState::load(&runner.state.id).unwrap();
15925 assert!(saved.candidate_preservation.contains_key(&'A'));
15926 assert!(!saved.candidates[0].empty);
15927 assert_eq!(
15928 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15929 "retain implementation work\n"
15930 );
15931 std::fs::remove_file(lock).unwrap();
15932 runner.state = saved;
15933 } else {
15934 assert_eq!(runner.state.status, RunStatus::Blocked);
15935 assert_eq!(runner.state.candidates[0].commits, 1);
15936 }
15937 runner.roles.implementers.clear();
15940 runner.state.config.disk.min_free_bytes = 0;
15941 runner.state.resource_block = None;
15942 runner.implement().await.unwrap();
15943 assert!(runner.state.candidate_preservation.is_empty());
15944 assert_eq!(runner.state.candidates[0].commits, 1);
15945 assert!(!runner.state.candidates[0].empty);
15946 assert!(git::is_clean(&wt).await.unwrap());
15947 assert!(
15948 crate::run::read_artifact(&runner.state, "cand-A.patch")
15949 .unwrap()
15950 .contains("retain implementation work")
15951 );
15952 }
15953 }
15954
15955 #[tokio::test]
15956 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15957 let (_tmp, mut runner) = preservation_resume_fixture().await;
15958 runner.implement().await.unwrap();
15960 assert!(runner.state.candidate_preservation.is_empty());
15961 assert!(
15962 !runner
15963 .state
15964 .dir()
15965 .join("artifacts/candidate-A-recovery")
15966 .exists()
15967 );
15968 runner.fold_losers().await.unwrap();
15969 assert!(!runner.state.candidates[0].worktree.exists());
15970 for schema in [14, crate::run::SCHEMA] {
15971 runner.state.schema = schema;
15972 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15973 runner.state.save().unwrap();
15974 runner.state = RunState::load(&runner.state.id).unwrap();
15975 runner.implement().await.unwrap();
15976 assert!(runner.state.candidate_preservation.is_empty());
15977 assert!(runner.state.candidates[0].folded);
15978 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15979 }
15980 }
15981
15982 #[tokio::test]
15983 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15984 let (_tmp, mut runner) = preservation_resume_fixture().await;
15985 runner.state.candidates.retain(|c| c.label == 'A');
15986 runner.state.tally = None;
15987 runner.state.judge_skipped = false;
15988 let candidate = &mut runner.state.candidates[0];
15989 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15990 candidate.failed = Some("deployment validation unavailable".into());
15991 std::fs::write(
15992 candidate.worktree.join("repair"),
15993 "future launcher policy\n",
15994 )
15995 .unwrap();
15996 runner
15997 .state
15998 .candidate_preservation
15999 .insert('A', "awaiting persistence".into());
16000 runner.persist_candidates().await.unwrap();
16001 let candidate = &runner.state.candidates[0];
16002 assert!(!candidate.empty);
16003 assert_eq!(candidate.commits, 1);
16004 assert_eq!(candidate.files, 1);
16005 assert!(candidate.failed.is_none());
16006 assert!(candidate.summary.contains("EPERM"));
16007 assert!(
16008 crate::run::read_artifact(&runner.state, "cand-A.patch")
16009 .unwrap()
16010 .contains("future launcher policy")
16011 );
16012 assert!(runner.state.candidate_preservation.is_empty());
16013 }
16014
16015 #[tokio::test]
16016 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16017 let (_tmp, mut runner) = preservation_resume_fixture().await;
16018 let wt = runner.state.candidates[0].worktree.clone();
16019 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16020 runner.state.schema = 14;
16021 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16022 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16023 let tally = serde_json::to_value(&runner.state.tally).unwrap();
16024 for _ in 0..2 {
16025 assert!(runner.implement().await.is_err());
16026 assert_eq!(runner.state.status, RunStatus::Blocked);
16027 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16028 assert!(
16029 runner.state.candidates[0].empty,
16030 "old verdict must remain unusable"
16031 );
16032 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16033 assert!(runner.state.judge_skipped);
16034 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16035 assert!(
16036 crate::run::read_artifact(&runner.state, "cand-A.patch")
16037 .unwrap()
16038 .contains("owned edits")
16039 );
16040 assert!(wt.exists());
16041 assert_eq!(
16042 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16043 .await
16044 .unwrap(),
16045 1
16046 );
16047 runner.state = RunState::load(&runner.state.id).unwrap();
16048 }
16049 }
16050
16051 #[tokio::test]
16052 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16053 let (_tmp, mut runner) = preservation_resume_fixture().await;
16054 let wt = runner.state.candidates[0].worktree.clone();
16055 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16056 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16057 let index_lock = PathBuf::from(
16058 git::git(
16059 &wt,
16060 &[
16061 "rev-parse",
16062 "--path-format=absolute",
16063 "--git-path",
16064 "index.lock",
16065 ],
16066 )
16067 .await
16068 .unwrap(),
16069 );
16070 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16071 runner.state.schema = 14;
16072 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16073 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16074 assert!(runner.implement().await.is_err());
16075 assert!(runner.after_implement().is_err());
16076 assert!(runner.fold_losers().await.is_err());
16077 assert_eq!(
16078 std::fs::read_to_string(wt.join("recovered")).unwrap(),
16079 "failed seat edits"
16080 );
16081 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16082 runner.state = RunState::load(&runner.state.id).unwrap();
16083 std::fs::remove_file(index_lock).unwrap();
16084 assert!(runner.implement().await.is_err());
16086 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16087 assert!(runner.state.candidates[0].empty);
16088 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16089 assert!(
16090 crate::run::read_artifact(&runner.state, "cand-A.patch")
16091 .unwrap()
16092 .contains("failed seat edits")
16093 );
16094 assert!(wt.exists());
16095 }
16096
16097 #[tokio::test]
16098 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16099 let (_tmp, mut runner) = preservation_resume_fixture().await;
16100 let repo = runner.state.repo.clone();
16101 let common = git::git(
16102 &repo,
16103 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16104 )
16105 .await
16106 .unwrap();
16107 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16108 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16109 assert!(runner.fold_losers().await.is_err());
16110 assert!(!runner.state.candidates[0].worktree.exists());
16111 assert!(!runner.state.candidates[0].folded);
16112 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16113 runner.state = RunState::load(&runner.state.id).unwrap();
16114 std::fs::remove_file(branch_lock).unwrap();
16115 let judge = runner.state.worktree_root().join("judge-1");
16116 git::worktree_add_detached(&repo, &judge, "main")
16117 .await
16118 .unwrap();
16119 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16120 runner.roles.judges = vec![spec("judge")];
16121 assert!(runner.fold_losers().await.is_err());
16122 runner.state = RunState::load(&runner.state.id).unwrap();
16123 assert!(
16124 runner.state.candidates[0].folded,
16125 "candidate progress saved before judge failure"
16126 );
16127 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16128 assert!(judge.join("uncommitted").exists());
16129 std::fs::remove_file(judge.join("uncommitted")).unwrap();
16130 runner.fold_losers().await.unwrap();
16131 assert!(!judge.exists());
16132 runner.fold_losers().await.unwrap();
16133 }
16134
16135 #[tokio::test]
16136 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16137 let (_tmp, mut runner) = preservation_resume_fixture().await;
16138 runner.state.candidates.retain(|c| c.label == 'B');
16139 runner.state.tally = None;
16140 runner.state.judge_skipped = false;
16141 runner.state.prep_complete = Some(true);
16142 runner.state.config.graph.advise = false;
16143 runner.state.config.graph.selected_implementer = Some("alpha".into());
16144 let wt = runner.state.candidates[0].worktree.clone();
16145 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16146 runner
16147 .state
16148 .candidate_preservation
16149 .insert('B', "awaiting persistence".into());
16150 let index_lock = git::git(
16151 &wt,
16152 &[
16153 "rev-parse",
16154 "--path-format=absolute",
16155 "--git-path",
16156 "index.lock",
16157 ],
16158 )
16159 .await
16160 .unwrap();
16161 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16162 runner.state.save().unwrap();
16163 assert!(runner.execute_graph().await.is_err());
16164 assert_eq!(runner.state.status, RunStatus::Blocked);
16165 assert!(runner.state.candidate_preservation.contains_key(&'B'));
16166 std::fs::remove_file(index_lock).unwrap();
16167 for _ in 0..2 {
16170 runner.state = RunState::load(&runner.state.id).unwrap();
16171 runner.pause.park();
16172 runner.execute_graph().await.unwrap();
16173 assert_eq!(runner.state.status, RunStatus::Implementing);
16174 assert!(runner.state.candidate_preservation.is_empty());
16175 assert_eq!(runner.state.candidates[0].commits, 2);
16176 assert!(runner.state.tally.is_none());
16177 }
16178 runner.implement().await.unwrap();
16179 assert_eq!(runner.state.status, RunStatus::Judging);
16180 runner.state.status = RunStatus::Blocked;
16182 runner.state.candidates[0].failed = Some("implementation failed".into());
16183 runner.state.save().unwrap();
16184 let error = runner.execute_graph().await.unwrap_err();
16185 assert!(
16186 error
16187 .to_string()
16188 .contains("selected implementation is held")
16189 );
16190 }
16191
16192 #[tokio::test]
16193 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16194 let (_tmp, mut runner) = preservation_resume_fixture().await;
16195 let repo = runner.state.repo.clone();
16196 let candidate = runner.state.candidates[0].clone();
16197 let registered = git::worktree_holding(&repo, &candidate.branch)
16200 .await
16201 .unwrap()
16202 .unwrap();
16203 assert_eq!(
16204 registered.canonicalize().unwrap(),
16205 candidate.worktree.canonicalize().unwrap()
16206 );
16207 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16208 assert_eq!(
16209 git::worktree_holding(&repo, &candidate.branch)
16210 .await
16211 .unwrap(),
16212 Some(registered)
16213 );
16214 assert!(
16215 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16216 "the stale registration initially keeps the branch checked out"
16217 );
16218 runner.fold_losers().await.unwrap();
16219 assert!(runner.state.candidates[0].folded);
16220 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16221 assert_eq!(
16222 git::worktree_holding(&repo, &candidate.branch)
16223 .await
16224 .unwrap(),
16225 None
16226 );
16227 runner.state = RunState::load(&runner.state.id).unwrap();
16228 runner.fold_losers().await.unwrap();
16229 }
16230
16231 #[tokio::test]
16232 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16233 let (_tmp, mut runner) = preservation_resume_fixture().await;
16234 let repo = runner.state.repo.clone();
16235 let candidate = runner.state.candidates[0].clone();
16236 let winner = runner.state.candidates[1].clone();
16237 let registered = git::worktree_holding(&repo, &candidate.branch)
16238 .await
16239 .unwrap()
16240 .unwrap();
16241 assert_eq!(
16242 registered.canonicalize().unwrap(),
16243 candidate.worktree.canonicalize().unwrap()
16244 );
16245 std::fs::remove_dir_all(&candidate.worktree).unwrap();
16246 assert_eq!(
16247 git::worktree_holding(&repo, &candidate.branch)
16248 .await
16249 .unwrap(),
16250 Some(registered)
16251 );
16252 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16253
16254 for _ in 0..2 {
16255 fold_run(&mut runner.state, false, &crate::run::home())
16256 .await
16257 .unwrap();
16258 runner.state = RunState::load(&runner.state.id).unwrap();
16259 assert!(runner.state.candidates[0].folded);
16260 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16261 assert_eq!(
16262 git::worktree_holding(&repo, &candidate.branch)
16263 .await
16264 .unwrap(),
16265 None
16266 );
16267 assert!(!runner.state.candidates[1].folded);
16268 assert!(winner.worktree.exists());
16269 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16270 }
16271 }
16272
16273 #[tokio::test]
16274 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16275 ask_test_home();
16276 let tmp = tempfile::tempdir().unwrap();
16277 let repo = tmp.path().join("repo");
16278 std::fs::create_dir(&repo).unwrap();
16279 init_repo(&repo);
16280 let mut runner = runner_at(RunStatus::Implementing);
16281 runner.state.repo = repo.clone();
16282 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16283 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16284 for (index, label) in ['A', 'B'].into_iter().enumerate() {
16285 let branch = format!("candidate-{label}");
16286 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16287 git::worktree_add_branch(&repo, &wt, &branch, "main")
16288 .await
16289 .unwrap();
16290 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16291 git::git(&wt, &["add", "README.md"]).await.unwrap();
16292 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16293 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16294 runner.state.candidates.push(Candidate {
16295 index,
16296 label,
16297 agent: "alpha".to_owned(),
16298 branch,
16299 worktree: wt,
16300 summary: "completed implementation".to_owned(),
16301 stat: String::new(),
16302 files: 0,
16303 commits: 0,
16304 empty: false,
16305 failed: None,
16306 verified_noop: None,
16307 duration_ms: 42,
16308 folded: false,
16309 });
16310 runner
16311 .state
16312 .candidate_preservation
16313 .insert(label, "awaiting persistence".to_owned());
16314 }
16315 let wt = runner.state.candidates[0].worktree.clone();
16316 let index_lock = git::git(
16317 &wt,
16318 &[
16319 "rev-parse",
16320 "--path-format=absolute",
16321 "--git-path",
16322 "index.lock",
16323 ],
16324 )
16325 .await
16326 .unwrap();
16327 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16328 runner.state.save().unwrap();
16329 assert!(runner.persist_candidates().await.is_err());
16330 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16331 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16332 assert_eq!(
16333 runner.state.candidates[1].commits, 1,
16334 "other results are still collected"
16335 );
16336 assert!(
16337 !runner.state.candidates[0].empty,
16338 "failed rescue is not no change"
16339 );
16340 assert_eq!(
16341 runner.state.candidates[0].summary,
16342 "completed implementation"
16343 );
16344 assert!(runner.after_implement().is_err());
16345 let home = tmp.path().join("fold-home");
16346 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16347 assert!(wt.join("new-file").exists());
16348 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16349 assert!(recovery.join("staged.patch").exists());
16350 assert!(recovery.join("unstaged.patch").exists());
16351 assert_eq!(
16352 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16353 "untracked edits\n"
16354 );
16355 runner.state.begin_pass();
16356 runner.state.end_pass_parked(ParkCause::Upgrade {
16357 from: "1.0".into(),
16358 to: Some("1.1".into()),
16359 });
16360 runner.state.save().unwrap();
16361 runner.state = RunState::load(&runner.state.id).unwrap();
16362 assert!(matches!(
16363 runner.state.passes.last().unwrap().end,
16364 Some(crate::run::PassEnd::Parked {
16365 cause: ParkCause::Upgrade { .. }
16366 })
16367 ));
16368 std::fs::remove_file(index_lock).unwrap();
16369 runner.implement().await.unwrap();
16371 assert!(runner.state.candidate_preservation.is_empty());
16372 assert!(
16373 runner
16374 .state
16375 .candidates
16376 .iter()
16377 .all(|c| c.commits == 1 && !c.empty)
16378 );
16379 assert!(git::is_clean(&wt).await.unwrap());
16380 assert!(
16381 crate::run::read_artifact(&runner.state, "cand-A.patch")
16382 .unwrap()
16383 .contains("later tracked edits")
16384 );
16385 }
16386
16387 #[tokio::test]
16388 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16389 ask_test_home();
16390 let tmp = tempfile::tempdir().unwrap();
16391 let repo = tmp.path().join("repo");
16392 std::fs::create_dir(&repo).unwrap();
16393 init_repo(&repo);
16394 let wt = tmp.path().join("candidate");
16395 git::worktree_add_branch(&repo, &wt, "candidate", "main")
16396 .await
16397 .unwrap();
16398 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16399 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16400 assert!(wt.join("uncommitted").exists());
16401 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16402 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16403 let mut runner = runner_at(RunStatus::Implementing);
16404 runner.state.repo = repo.clone();
16405 runner.state.base_commit = "missing-base".to_owned();
16406 runner.state.candidates.push(Candidate {
16407 index: 0,
16408 label: 'A',
16409 agent: "alpha".into(),
16410 branch: "candidate".into(),
16411 worktree: wt.clone(),
16412 summary: "completed".into(),
16413 stat: String::new(),
16414 files: 0,
16415 commits: 0,
16416 empty: false,
16417 failed: None,
16418 verified_noop: None,
16419 duration_ms: 1,
16420 folded: false,
16421 });
16422 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16423 assert!(
16424 fold_run(&mut runner.state, true, &tmp.path().join("home"))
16425 .await
16426 .is_err()
16427 );
16428 assert!(!runner.state.candidates[0].folded);
16429 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16430 std::fs::remove_file(wt.join("uncommitted")).unwrap();
16431 runner
16432 .state
16433 .candidate_preservation
16434 .insert('A', "pending".into());
16435 assert!(runner.persist_candidates().await.is_err());
16436 assert!(!runner.state.candidates[0].empty);
16437 assert!(!runner.state.candidates[0].folded);
16438 assert!(wt.exists());
16439 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16440 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16441 std::fs::create_dir_all(&patch_path).unwrap();
16442 assert!(
16443 runner.persist_candidates().await.is_err(),
16444 "artifact failure stays pending"
16445 );
16446 assert!(runner.state.candidate_preservation.contains_key(&'A'));
16447 assert!(!runner.state.candidates[0].empty);
16448 std::fs::remove_dir(patch_path).unwrap();
16449 runner.persist_candidates().await.unwrap();
16450 assert!(
16451 runner.state.candidates[0].empty,
16452 "only successful inspection confirms no change"
16453 );
16454 }
16455
16456 fn init_repo(dir: &Path) {
16457 let run = |args: &[&str]| {
16458 let out = std::process::Command::new("git")
16459 .args(args)
16460 .current_dir(dir)
16461 .quiet()
16462 .output()
16463 .expect("spawn git");
16464 assert!(
16465 out.status.success(),
16466 "git {args:?} failed: {}",
16467 String::from_utf8_lossy(&out.stderr)
16468 );
16469 };
16470 run(&["init", "-b", "main"]);
16471 run(&["config", "user.name", "magi test"]);
16472 run(&["config", "user.email", "magi@example.com"]);
16473 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16474 run(&["add", "-A"]);
16475 run(&["commit", "-m", "init"]);
16476 }
16477
16478 fn ask_test_home() {
16486 crate::run::pin_test_home();
16487 }
16488
16489 #[test]
16490 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16491 ask_test_home();
16492 let mut runner = runner_at(RunStatus::Prep);
16493 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16494 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16497 let error = guard
16498 .finish(
16499 &mut runner.state,
16500 Err(
16501 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16502 .into(),
16503 ),
16504 )
16505 .unwrap_err();
16506 assert_eq!(
16507 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16508 std::io::ErrorKind::PermissionDenied
16509 );
16510 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16511 let message = format!("{error:#}");
16512 assert!(message.contains("original refusal"));
16513 assert!(!message.contains("could not save terminal"));
16514 assert_eq!(
16515 error.chain().count(),
16516 2,
16517 "only startup attribution wraps the original error"
16518 );
16519 let saved = RunState::load(&runner.state.id).unwrap();
16520 assert_eq!(saved.status, RunStatus::Blocked);
16521 assert!(!saved.startup.unwrap().complete);
16522 }
16523
16524 #[test]
16525 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16526 ask_test_home();
16527 let mut runner = runner_at(RunStatus::Prep);
16528 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16529 let id = runner.state.id.clone();
16530 drop(guard);
16531 let saved = RunState::load(&id).unwrap();
16532 assert_eq!(saved.status, RunStatus::Blocked);
16533 assert!(!saved.startup.as_ref().unwrap().complete);
16534 assert!(
16535 saved
16536 .startup
16537 .unwrap()
16538 .failure
16539 .unwrap()
16540 .contains("interrupted")
16541 );
16542 assert!(Runner::resume(&id).is_err());
16543 }
16544
16545 #[test]
16546 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16547 ask_test_home();
16548 let mut runner = runner_at(RunStatus::Prep);
16549 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16550 let error = guard
16551 .finish(
16552 &mut runner.state,
16553 Err(
16554 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16555 .into(),
16556 ),
16557 )
16558 .unwrap_err();
16559 assert_eq!(
16560 error.downcast_ref::<std::io::Error>().unwrap().kind(),
16561 std::io::ErrorKind::PermissionDenied
16562 );
16563 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16564 let mut task = crate::queue::Task::new(
16565 "review".into(),
16566 "criteria".into(),
16567 runner.state.repo.clone(),
16568 crate::queue::Source::Human,
16569 );
16570 task.record_startup_failure(failure.clone());
16571 task.record_startup_failure(failure.clone());
16572 assert_eq!(task.startup_failures.len(), 1);
16573 assert!(task.runs.is_empty());
16574 assert_eq!(task.attempts, 0);
16575 }
16576
16577 #[cfg(unix)]
16578 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16579 ask_test_home();
16580 let tmp = tempfile::tempdir().unwrap();
16581 let repo = tmp.path().join("repo");
16582 std::fs::create_dir(&repo).unwrap();
16583 init_repo(&repo);
16584 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16585 let origin = tmp.path().join("origin.git");
16586 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16587 .await
16588 .unwrap();
16589 git::git(
16590 &repo,
16591 &["remote", "add", "origin", origin.to_str().unwrap()],
16592 )
16593 .await
16594 .unwrap();
16595 git::git(&repo, &["checkout", "-b", "fix/retained"])
16596 .await
16597 .unwrap();
16598 std::fs::write(repo.join("retained"), "retained change").unwrap();
16599 git::git(&repo, &["add", "."]).await.unwrap();
16600 git::git(&repo, &["commit", "-m", "retained implementation"])
16601 .await
16602 .unwrap();
16603 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16604 git::git(&repo, &["checkout", "main"]).await.unwrap();
16605 std::fs::write(repo.join("base-change"), "new base").unwrap();
16606 git::git(&repo, &["add", "."]).await.unwrap();
16607 git::git(&repo, &["commit", "-m", "base moved"])
16608 .await
16609 .unwrap();
16610 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16611 git::git(&repo, &["checkout", "fix/retained"])
16612 .await
16613 .unwrap();
16614 let mut runner = runner_at(RunStatus::Voting);
16615 runner.state.repo = repo.clone();
16616 runner.state.base_commit = base;
16617 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16618 runner.state.config.disk.min_free_bytes = 0;
16619 runner.state.config.graph.review_rounds = 1;
16620 runner.state.config.verify.e2e = vec!["true".into()];
16621 runner.state.config.verify.gate = vec!["true".into()];
16622 runner.state.candidates = vec![Candidate {
16623 index: 0,
16624 label: 'A',
16625 agent: "fixture".into(),
16626 branch: "fix/retained".into(),
16627 worktree: repo.clone(),
16628 summary: String::new(),
16629 stat: String::new(),
16630 files: 1,
16631 commits: 1,
16632 empty: false,
16633 failed: None,
16634 verified_noop: None,
16635 duration_ms: 0,
16636 folded: false,
16637 }];
16638 runner.state.tally = Some(Tally {
16639 first_choice: BTreeMap::new(),
16640 borda: BTreeMap::new(),
16641 winner: 'A',
16642 rankings: 0,
16643 unanimous_initial: true,
16644 deliberated: false,
16645 changed_votes: 0,
16646 unanimous_final: true,
16647 tie_break: None,
16648 judges: 0,
16649 present: 0,
16650 quorum: 0,
16651 met_quorum: true,
16652 uncontested: Some("review-only".into()),
16653 });
16654 let mut task = crate::queue::Task::new(
16655 "fixture".into(),
16656 "fixture".into(),
16657 repo,
16658 crate::queue::Source::Human,
16659 );
16660 task.link_run(&runner.state.id);
16661 crate::queue::Queue::open().put(&mut task).unwrap();
16662 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16663 runner.state.base_sync = Some(BaseSync {
16664 tip: String::new(),
16665 behind: 1,
16666 attempts: BASE_SYNC_ROUNDS,
16667 conflict: None,
16668 already_in: None,
16669 });
16670 runner.sync_to_base().await.unwrap();
16671 assert_eq!(runner.state.status, RunStatus::Blocked);
16672 (tmp, runner, head, task.id)
16673 }
16674
16675 #[cfg(unix)]
16676 #[tokio::test]
16677 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16678 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16679 if std::env::var_os(CHILD).is_none() {
16680 let home = tempfile::tempdir().unwrap();
16681 let output = std::process::Command::new(std::env::current_exe().unwrap())
16682 .args([
16683 "--exact",
16684 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16685 "--nocapture",
16686 ])
16687 .env(CHILD, "1")
16688 .env("MAGI_HOME", home.path())
16689 .env("MAGI_WORKER_HOME", home.path())
16690 .env("MAGI_WORKER_ID", "abcdef")
16691 .output()
16692 .unwrap();
16693 assert!(
16694 output.status.success()
16695 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16696 "{}\n{}",
16697 String::from_utf8_lossy(&output.stdout),
16698 String::from_utf8_lossy(&output.stderr)
16699 );
16700 return;
16701 }
16702 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16703 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16704 let home = crate::run::home();
16705 let queue = crate::queue::Queue::open();
16706 let mut owner = queue.get(&task).unwrap();
16707 owner.release_operator("test-release", None);
16708 owner.start(runner.state.id.clone());
16709 queue.put(&mut owner).unwrap();
16710 runner.state.driver_exited = true;
16711 runner.state.task_instruction = Some(owner.instruction.clone());
16712 runner.state.save().unwrap();
16713 let _claim = queue.claim(&task).unwrap();
16714 let reserve = |id: &str| {
16715 let directory = home.join("workers/jobs").join(id);
16716 std::fs::create_dir_all(&directory).unwrap();
16717 let job = crate::worker::Job {
16718 id: id.into(),
16719 sequence: 1,
16720 kind: "task".into(),
16721 key: format!("task:{task}"),
16722 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16723 generation: "fixture-generation".into(),
16724 version: None,
16725 status: crate::worker::Status::Running,
16726 result: None,
16727 pid: Some(std::process::id()),
16728 created_at: 0,
16729 finished_at: None,
16730 };
16731 std::fs::write(
16732 directory.join("job.json"),
16733 serde_json::to_vec(&job).unwrap(),
16734 )
16735 .unwrap();
16736 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16737 };
16738 let _own = reserve("abcdef");
16739 let other = reserve("fedcba");
16740 assert!(
16741 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16742 .await
16743 .is_err()
16744 );
16745 assert!(
16746 RunState::load(&runner.state.id)
16747 .unwrap()
16748 .base_sync_recovery
16749 .authorizations
16750 .is_empty()
16751 );
16752 drop(other);
16753 assert!(crate::daemon::is_working_on(
16754 &home,
16755 &runner.state.id,
16756 Timestamp::now()
16757 ));
16758 assert!(
16759 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16760 );
16761 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16762 .await
16763 .unwrap();
16764 assert!(
16765 RunState::load(&runner.state.id)
16766 .unwrap()
16767 .base_recovery_pending()
16768 );
16769 }
16770
16771 #[cfg(unix)]
16772 #[tokio::test]
16773 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16774 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16775 let home = crate::run::home();
16776 runner.state.driver_pid = Some(std::process::id());
16777 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16778 runner.state.driver_exited = false;
16779 runner.state.begin_pass();
16780 runner.state.save().unwrap();
16781 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16782 driver.record(&runner.state).unwrap();
16783 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16784 drop(driver);
16786 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16787 assert_eq!(
16788 runner.state.liveness_under(&home, false),
16789 crate::run::Liveness::Dead
16790 );
16791 let queue = crate::queue::Queue::open();
16792 let _claim = queue.claim(&task).unwrap();
16793 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16794 .await
16795 .unwrap();
16796 let disk = RunState::load(&runner.state.id).unwrap();
16797 assert!(!disk.driver_exited);
16798 assert!(disk.base_recovery_pending());
16799 }
16800
16801 #[cfg(unix)]
16802 #[tokio::test]
16803 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16804 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16805 if std::env::var_os(CHILD).is_none() {
16806 use std::os::unix::fs::PermissionsExt;
16807 let tmp = tempfile::tempdir().unwrap();
16808 let gh = tmp.path().join("gh");
16809 std::fs::write(&gh, r#"#!/bin/sh
16810case "$1 $2" in
16811 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16812 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16813 *) exit 2 ;;
16814esac
16815"#).unwrap();
16816 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16817 let mut paths = vec![tmp.path().to_path_buf()];
16818 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16819 let output = std::process::Command::new(std::env::current_exe().unwrap())
16820 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16821 .env(CHILD, "1")
16822 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16823 .env("PATH", std::env::join_paths(paths).unwrap())
16824 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16825 .output().unwrap();
16826 assert!(
16827 output.status.success()
16828 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16829 "{}\n{}",
16830 String::from_utf8_lossy(&output.stdout),
16831 String::from_utf8_lossy(&output.stderr)
16832 );
16833 return;
16834 }
16835 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16836 let repo = runner.state.repo.clone();
16837 git::git(&repo, &["checkout", "main"]).await.unwrap();
16838 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16839 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16840 git::git(&repo, &["checkout", "fix/retained"])
16841 .await
16842 .unwrap();
16843 runner.state.pr = Some(crate::run::PrRecord {
16844 url: "https://example.test/project/pull/1".into(),
16845 number: 1,
16846 state: "open".into(),
16847 checks: "unknown".into(),
16848 round: 0,
16849 rounds: 1,
16850 red_at_merge: Vec::new(),
16851 });
16852 runner.sync_to_base().await.unwrap();
16853 assert_eq!(runner.state.status, RunStatus::Blocked);
16854 assert_eq!(
16855 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16856 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16857 );
16858 assert!(runner.state.base_sync_closure_pending());
16859 assert!(crate::handover::can_resume(&runner.state));
16860 let queue = crate::queue::Queue::open();
16861 assert!(!crate::base_sync_recovery::held_task(
16862 &queue,
16863 &queue.get(&task).unwrap()
16864 ));
16865 assert!(runner.state.authorize_base_sync(&task).is_err());
16866 runner.state.base_sync_recovery.stop = None;
16868 runner.state.status = RunStatus::Voting;
16869 runner.state.gate_ran = true;
16870 runner.state.config.graph.review_rounds = 0;
16871 runner.state.save().unwrap();
16872 assert!(crate::handover::can_resume(&runner.state));
16873 runner.state = RunState::load(&runner.state.id).unwrap();
16874 runner.execute_graph().await.unwrap();
16875 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16876 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16877 assert_eq!(
16878 runner.state.base_sync.as_ref().unwrap().attempts,
16879 BASE_SYNC_ROUNDS
16880 );
16881 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16882 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16883 assert!(runner.state.reviews.is_empty());
16884 assert!(runner.state.gate.is_empty());
16885 }
16886
16887 #[cfg(unix)]
16888 #[tokio::test]
16889 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16890 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16891 runner.state.driver_exited = true;
16892 runner.state.save().unwrap();
16893 let home = crate::run::home();
16894 let queue = crate::queue::Queue::open();
16895 let _claim = queue.claim(&task).unwrap();
16896 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16897 assert!(
16898 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16899 .await
16900 .is_err()
16901 );
16902 drop(lease);
16903 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16904 assert!(
16905 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16906 .await
16907 .is_err()
16908 );
16909 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16910 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16911 .await
16912 .unwrap();
16913 let disk = RunState::load(&runner.state.id).unwrap();
16914 assert!(disk.base_recovery_pending());
16915 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16916 assert!(
16917 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16918 .await
16919 .is_err()
16920 );
16921 assert_eq!(
16922 RunState::load(&runner.state.id)
16923 .unwrap()
16924 .base_sync_recovery
16925 .authorizations
16926 .len(),
16927 1
16928 );
16929 let mut owner = queue.get(&task).unwrap();
16930 owner.runs.push("20260101-000000-ffff".into());
16931 queue.put(&mut owner).unwrap();
16932 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16933 .await
16934 .unwrap_err();
16935 assert!(error.to_string().contains("run task ownership changed"));
16936 }
16937
16938 #[cfg(unix)]
16939 #[tokio::test]
16940 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16941 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16942 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16943 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16944 legacy["status"] = serde_json::json!("voting");
16945 runner.state = serde_json::from_value(legacy).unwrap();
16946 runner.state.save().unwrap();
16947 for _ in 0..3 {
16948 runner.execute_graph().await.unwrap();
16949 assert_eq!(runner.state.status, RunStatus::Blocked);
16950 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16951 assert!(!crate::handover::can_resume(&runner.state));
16952 assert_eq!(
16953 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16954 head
16955 );
16956 assert!(runner.state.reviews.is_empty());
16957 assert!(!runner.state.gate_ran);
16958 }
16959 let notice = crate::notices::run_ended(&runner.state).unwrap();
16960 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16961 runner.state.begin_pass();
16962 runner.state.status = RunStatus::Voting;
16963 runner.deliberate().await.unwrap();
16964 runner.vote().await.unwrap();
16965 runner.finish_after_tally().await.unwrap();
16966 assert_eq!(runner.state.status, RunStatus::Blocked);
16967 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16968 assert_eq!(
16969 crate::notices::run_ended(&runner.state).unwrap().since,
16970 notice.since
16971 );
16972 }
16973
16974 #[cfg(unix)]
16975 #[tokio::test]
16976 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16977 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16978 for detail in [
16979 "fix/retained conflicts with origin/main and did not rebase: conflict",
16980 "rebased fix/retained locally but origin refused the push",
16981 "unknown failure",
16982 ] {
16983 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16984 runner.state.record_base_sync_stop(&head);
16985 assert!(runner.state.authorize_base_sync(&task).is_err());
16986 runner.state.status = RunStatus::Voting;
16987 runner.execute_graph().await.unwrap();
16988 assert_eq!(runner.state.status, RunStatus::Blocked);
16989 assert!(!crate::handover::can_resume(&runner.state));
16990 assert_eq!(
16991 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16992 head
16993 );
16994 }
16995 }
16996
16997 #[cfg(unix)]
16998 #[tokio::test]
16999 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17000 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17001 runner.state.authorize_base_sync(&task).unwrap();
17002 assert!(runner.state.consume_base_recovery(&head));
17003 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17004 runner.state.reserve_base_operation(&head, &tip);
17005 runner.state.save().unwrap();
17006 runner.state = RunState::load(&runner.state.id).unwrap();
17007 assert!(!runner.state.consume_base_recovery(&head));
17008 runner.sync_to_base().await.unwrap();
17009 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17010 assert_eq!(runner.state.base_episode_used(), 1);
17011 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17012 assert!(
17013 runner
17014 .state
17015 .base_sync_recovery
17016 .operations
17017 .last()
17018 .unwrap()
17019 .finished
17020 );
17021 assert!(!runner.state.base_sync_blocked());
17022 runner.sync_to_base().await.unwrap();
17023 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17024 }
17025
17026 #[cfg(unix)]
17027 #[tokio::test]
17028 async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17029 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17030 let original = runner
17032 .state
17033 .base_sync
17034 .as_ref()
17035 .unwrap()
17036 .conflict
17037 .clone()
17038 .unwrap();
17039 runner.state.base_sync.as_mut().unwrap().conflict = Some(
17040 original
17041 .replace("synchronization(s)", "rebase(s)")
17042 .replace("synchronizing", "rebasing"),
17043 );
17044 runner.state.base_sync_recovery = Default::default();
17045 runner.state.record_base_sync_stop(&head);
17046 assert_eq!(
17047 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17048 crate::base_sync_recovery::StopKind::Exhausted
17049 );
17050 runner.state.authorize_base_sync(&task).unwrap();
17051 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17052 let auth = legacy["base_sync_recovery"]["authorizations"][0]
17053 .as_object_mut()
17054 .unwrap();
17055 auth.remove("interval");
17056 auth.remove("scope");
17057 runner.state = serde_json::from_value(legacy).unwrap();
17058 assert!(runner.state.consume_base_recovery(&head));
17059 assert_eq!(runner.state.base_episode_used(), 3);
17060 runner.sync_to_base().await.unwrap();
17061 assert_eq!(runner.state.base_episode_used(), 4);
17062 runner.state.resource_block = Some(crate::run::ResourceBlock {
17063 node: "gate".into(),
17064 kind: "cache".into(),
17065 detail: "legacy wait".into(),
17066 at: Timestamp::now(),
17067 cause: crate::run::ResourceBlockCause::Unrecorded,
17068 });
17069 runner.state.save().unwrap();
17070 assert!(
17071 runner
17072 .state
17073 .base_sync_recovery
17074 .episodes
17075 .last()
17076 .unwrap()
17077 .boundary
17078 .is_none()
17079 );
17080 runner.state.resource_block = None;
17081 advance_base_fixture(&runner).await;
17082 runner.sync_to_base().await.unwrap();
17083 assert!(runner.state.base_sync_blocked());
17084 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17085 }
17086
17087 #[cfg(unix)]
17088 async fn advance_base_fixture(runner: &Runner) {
17089 let repo = &runner.state.repo;
17090 for args in [
17091 vec!["checkout", "main"],
17092 vec!["commit", "--allow-empty", "-m", "base advanced"],
17093 vec!["push", "origin", "main"],
17094 vec!["checkout", "fix/retained"],
17095 ] {
17096 git::git(repo, &args).await.unwrap();
17097 }
17098 }
17099
17100 #[cfg(unix)]
17101 fn fresh_base_interval_fixture(runner: &mut Runner) {
17102 runner.state.base_sync.as_mut().unwrap().attempts = 0;
17103 runner.state.base_sync.as_mut().unwrap().conflict = None;
17104 runner.state.base_sync_recovery = Default::default();
17105 runner.state.status = RunStatus::Voting;
17106 }
17107
17108 #[cfg(unix)]
17109 #[tokio::test]
17110 async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17111 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17112 fresh_base_interval_fixture(&mut runner);
17113 runner.state.begin_pass();
17114 runner.sync_to_base().await.unwrap();
17115 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17116 runner.state.resource_block = Some(crate::run::ResourceBlock {
17117 node: "gate".into(),
17118 kind: "build cache".into(),
17119 detail: "busy".into(),
17120 at: Timestamp::now(),
17121 cause: crate::run::ResourceBlockCause::Timeout {
17122 waited_secs: 1,
17123 limit_secs: 1,
17124 },
17125 });
17126 runner.state.save().unwrap();
17127 let queue = crate::queue::Queue::open();
17128 let mut owner = queue.get(&task).unwrap();
17129 runner.state.task_instruction = Some(owner.instruction.clone());
17130 owner.release_operator("fixture-owner", None);
17131 queue.put(&mut owner).unwrap();
17132 let approval = owner.resumption.clone().unwrap();
17133 runner.state.resource_block = None;
17134 runner.state.save().unwrap();
17135 runner.state = RunState::load(&runner.state.id).unwrap();
17136 advance_base_fixture(&runner).await;
17137 runner.sync_to_base().await.unwrap();
17138 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17139 assert_eq!(runner.state.base_episode_used(), 1);
17140 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17141 assert!(
17142 runner.state.base_sync_recovery.episodes[0]
17143 .boundary
17144 .as_ref()
17145 .unwrap()
17146 .consumed
17147 );
17148 assert!(
17149 !runner
17150 .state
17151 .renew_base_episode(&task, &approval, &head, "base")
17152 );
17153 for _ in 0..3 {
17154 owner.release_operator("fixture-owner", None);
17155 queue.put(&mut owner).unwrap();
17156 runner.state = RunState::load(&runner.state.id).unwrap();
17157 runner.state.begin_pass();
17158 runner.sync_to_base().await.unwrap();
17159 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17160 assert_eq!(runner.state.base_episode_used(), 1);
17161 }
17162 runner.state.resource_block = Some(crate::run::ResourceBlock {
17163 node: "gate".into(),
17164 kind: "disk pressure".into(),
17165 detail: "free space required".into(),
17166 at: Timestamp::now(),
17167 cause: crate::run::ResourceBlockCause::Other,
17168 });
17169 runner.state.end_pass_parked(ParkCause::Interrupt {
17170 reason: "disk pressure".into(),
17171 });
17172 runner.state.save().unwrap();
17173 owner.release_operator("fixture-owner-after-disk", None);
17174 queue.put(&mut owner).unwrap();
17175 runner.state.resource_block = None;
17176 advance_base_fixture(&runner).await;
17177 runner.sync_to_base().await.unwrap();
17178 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17179 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17180 assert_eq!(runner.state.base_episode_used(), 1);
17181 assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17182 }
17183
17184 #[cfg(unix)]
17185 #[tokio::test]
17186 async fn base_recovery_continuous_advancement_stops_at_four() {
17187 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17188 fresh_base_interval_fixture(&mut runner);
17189 for spent in 1..=BASE_SYNC_ROUNDS {
17190 runner.sync_to_base().await.unwrap();
17191 assert!(!runner.state.base_sync_blocked());
17192 assert_eq!(runner.state.base_episode_used(), spent);
17193 advance_base_fixture(&runner).await;
17194 }
17195 runner.sync_to_base().await.unwrap();
17196 assert_eq!(
17197 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17198 crate::base_sync_recovery::StopKind::Exhausted
17199 );
17200 let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17201 assert!(stop.unverified_sync);
17202 let action = runner.state.base_sync_next_action();
17203 assert!(action.contains("never reached its gate"));
17204 assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17205 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17206 runner.state = RunState::load(&runner.state.id).unwrap();
17207 runner.sync_to_base().await.unwrap();
17208 assert_eq!(runner.state.base_episode_used(), 4);
17209 assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17210 }
17211
17212 #[cfg(unix)]
17213 #[tokio::test]
17214 async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17215 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17216 runner.state.ensure_base_episode();
17217 runner.state.record_base_progress(&head);
17219 runner.state.base_sync_recovery.stop = None;
17220 runner.state.record_base_sync_stop(&head);
17221 assert!(
17222 !runner
17223 .state
17224 .base_sync_recovery
17225 .stop
17226 .as_ref()
17227 .unwrap()
17228 .unverified_sync
17229 );
17230
17231 runner.state.finish_base_operation(Some(&head));
17232 runner.state.gate_ran = true;
17234 runner.state.gate_head = Some("older-head".into());
17235 runner.state.base_sync_recovery.stop = None;
17236 runner.state.record_base_sync_stop(&head);
17237 runner.state.save().unwrap();
17238 runner.state = RunState::load(&runner.state.id).unwrap();
17239 assert!(
17240 runner
17241 .state
17242 .base_sync_recovery
17243 .stop
17244 .as_ref()
17245 .unwrap()
17246 .unverified_sync
17247 );
17248
17249 runner.state.gate_head = Some(head.clone());
17250 runner.state.base_sync_recovery.stop = None;
17251 runner.state.record_base_sync_stop(&head);
17252 assert!(
17253 !runner
17254 .state
17255 .base_sync_recovery
17256 .stop
17257 .as_ref()
17258 .unwrap()
17259 .unverified_sync
17260 );
17261 assert!(
17262 !runner
17263 .state
17264 .base_sync_next_action()
17265 .contains("never reached its gate")
17266 );
17267
17268 let mut legacy = serde_json::to_value(&runner.state).unwrap();
17270 legacy["base_sync_recovery"]["stop"]
17271 .as_object_mut()
17272 .unwrap()
17273 .remove("unverified_sync");
17274 legacy["base_sync_recovery"]["episodes"][0]
17275 .as_object_mut()
17276 .unwrap()
17277 .remove("last_sync_head");
17278 runner.state = serde_json::from_value(legacy).unwrap();
17279 assert!(
17280 !runner
17281 .state
17282 .base_sync_recovery
17283 .stop
17284 .as_ref()
17285 .unwrap()
17286 .unverified_sync
17287 );
17288 assert!(
17289 runner.state.base_sync_recovery.episodes[0]
17290 .last_sync_head
17291 .is_none()
17292 );
17293 }
17294
17295 #[cfg(unix)]
17296 #[tokio::test]
17297 async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17298 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17299 runner.state.authorize_base_sync(&task).unwrap();
17300 assert!(
17301 runner
17302 .state
17303 .base_sync_next_action()
17304 .contains("Approval recorded")
17305 );
17306 assert!(
17307 runner
17308 .state
17309 .base_sync_next_action()
17310 .contains("4/4 in episode 0, 4 lifetime")
17311 );
17312 assert!(runner.state.consume_base_recovery(&head));
17313 assert!(
17314 runner
17315 .state
17316 .base_sync_next_action()
17317 .contains("0/4 in episode 1, 4 lifetime")
17318 );
17319 }
17320
17321 #[cfg(unix)]
17322 #[tokio::test]
17323 async fn base_recovery_changed_head_invalidates_without_spending() {
17324 let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17325 runner.state.authorize_base_sync(&task).unwrap();
17326 git::git(
17327 &runner.state.repo,
17328 &["commit", "--allow-empty", "-m", "external head change"],
17329 )
17330 .await
17331 .unwrap();
17332 runner.sync_to_base().await.unwrap();
17333 let a = runner
17334 .state
17335 .base_sync_recovery
17336 .authorizations
17337 .last()
17338 .unwrap();
17339 assert!(a.consumed);
17340 assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17341 runner.state = RunState::load(&runner.state.id).unwrap();
17342 assert!(
17343 runner
17344 .state
17345 .base_sync_next_action()
17346 .contains("Approval invalidated")
17347 );
17348 assert!(!runner.state.base_recovery_pending());
17349 assert!(runner.state.base_sync_blocked());
17350 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17351 }
17352
17353 #[cfg(unix)]
17354 #[tokio::test]
17355 async fn base_recovery_stale_fetched_base_invalidates_approval() {
17356 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17357 runner.state.driver_exited = true;
17358 runner.state.save().unwrap();
17359 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17360 .await
17361 .unwrap();
17362 runner.state = RunState::load(&runner.state.id).unwrap();
17363 advance_base_fixture(&runner).await;
17364 runner.sync_to_base().await.unwrap();
17365 let a = runner
17366 .state
17367 .base_sync_recovery
17368 .authorizations
17369 .last()
17370 .unwrap();
17371 assert!(
17372 a.consumed
17373 && a.invalidated
17374 .as_deref()
17375 .unwrap()
17376 .contains("fetched base changed")
17377 );
17378 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17379 assert_eq!(
17380 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17381 head
17382 );
17383 Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17384 .await
17385 .unwrap();
17386 runner.state = RunState::load(&runner.state.id).unwrap();
17387 runner.sync_to_base().await.unwrap();
17388 assert!(!runner.state.base_sync_blocked());
17389 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17390 }
17391
17392 #[cfg(unix)]
17393 #[tokio::test]
17394 async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17395 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17396 runner.state.authorize_base_sync(&task).unwrap();
17397 assert!(runner.state.consume_base_recovery(&head));
17398 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17399 runner.state.reserve_base_operation(&head, &tip);
17400 runner.state.save().unwrap();
17401 let scratch = runner.state.dir().join("base-sync");
17402 assert_eq!(
17403 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17404 .await
17405 .unwrap(),
17406 crate::rebase::Rebased::Applied
17407 );
17408 runner.state = RunState::load(&runner.state.id).unwrap();
17409 runner.sync_to_base().await.unwrap();
17410 assert!(
17411 runner
17412 .state
17413 .base_sync_recovery
17414 .operations
17415 .last()
17416 .unwrap()
17417 .finished
17418 );
17419 assert_eq!(runner.state.base_episode_used(), 1);
17420 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17421 assert!(!runner.state.base_sync_blocked());
17422 }
17423
17424 #[cfg(unix)]
17425 #[tokio::test]
17426 async fn base_recovery_crash_preserves_untracked_files() {
17427 for ignored in [false, true] {
17428 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17429 runner.state.authorize_base_sync(&task).unwrap();
17430 assert!(runner.state.consume_base_recovery(&head));
17431 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17432 runner.state.reserve_base_operation(&head, &tip);
17433 runner.state.save().unwrap();
17434 let scratch = runner.state.dir().join("base-sync");
17435 assert_eq!(
17436 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17437 .await
17438 .unwrap(),
17439 crate::rebase::Rebased::Applied
17440 );
17441 let repo = &runner.state.repo;
17442 let result = git::rev_parse(repo, "HEAD").await.unwrap();
17443 assert_ne!(result, head);
17444 if ignored {
17445 let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17446 .await
17447 .unwrap();
17448 std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17449 }
17450 std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17451 assert!(
17452 git::git_raw(repo, &["diff", "--quiet", &head])
17453 .await
17454 .unwrap()
17455 .ok()
17456 );
17457 runner.state = RunState::load(&runner.state.id).unwrap();
17459 runner.sync_to_base().await.unwrap();
17460 assert!(runner.state.base_sync_blocked());
17461 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17462 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17463 assert_eq!(
17464 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17465 result
17466 );
17467 assert_eq!(
17468 std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17469 "retained work\n"
17470 );
17471 assert!(
17472 git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17473 .await
17474 .unwrap()
17475 .ok()
17476 );
17477 }
17478 }
17479
17480 #[cfg(unix)]
17481 #[tokio::test]
17482 async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17483 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17484 runner.state.authorize_base_sync(&task).unwrap();
17485 assert!(runner.state.consume_base_recovery(&head));
17486 let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17487 runner.state.reserve_base_operation(&head, &tip);
17488 runner.state.save().unwrap();
17489 let scratch = runner.state.dir().join("base-sync");
17490 crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17491 .await
17492 .unwrap();
17493 git::sync_to_head(&runner.state.repo).await.unwrap();
17494 std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17495 git::git(&runner.state.repo, &["add", "foreign"])
17496 .await
17497 .unwrap();
17498 git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17499 .await
17500 .unwrap();
17501 let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17502 runner.state = RunState::load(&runner.state.id).unwrap();
17503 runner.sync_to_base().await.unwrap();
17504 assert!(runner.state.base_sync_blocked());
17505 assert!(
17506 !runner
17507 .state
17508 .base_sync_recovery
17509 .operations
17510 .last()
17511 .unwrap()
17512 .finished
17513 );
17514 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17515 assert_eq!(
17516 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17517 foreign_head
17518 );
17519 }
17520
17521 #[cfg(unix)]
17522 async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17523 {
17524 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17525 fresh_base_interval_fixture(&mut runner);
17526 let repo = &runner.state.repo;
17527 git::git(repo, &["checkout", "main"]).await.unwrap();
17528 std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17529 git::git(repo, &["add", "retained"]).await.unwrap();
17530 git::git(repo, &["commit", "-m", "upstream retained change"])
17531 .await
17532 .unwrap();
17533 git::git(repo, &["push", "origin", "main"]).await.unwrap();
17534 let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17535 git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17536 let scratch = runner.state.dir().join("base-sync");
17537 assert!(matches!(
17538 git::rebase_start(repo, &scratch, "fix/retained", &tip)
17539 .await
17540 .unwrap(),
17541 git::RebaseStart::Conflicted(_)
17542 ));
17543 runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17544 agent: "fixture".into(),
17545 paths: vec!["retained".into()],
17546 from: Some(head.clone()),
17547 stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17548 index_outside: None,
17549 finished: false,
17550 error: None,
17551 launcher: None,
17552 infra: None,
17553 outage: None,
17554 });
17555 (tmp, runner, head, tip, scratch)
17556 }
17557
17558 #[cfg(unix)]
17559 #[tokio::test]
17560 async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17561 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17562 std::fs::write(
17565 scratch.join("retained"),
17566 "upstream and retained resolution\n",
17567 )
17568 .unwrap();
17569 runner.state.resource_block = Some(crate::run::ResourceBlock {
17570 node: "rebase".into(),
17571 kind: "repository coordination".into(),
17572 detail: "busy".into(),
17573 at: Timestamp::now(),
17574 cause: crate::run::ResourceBlockCause::Other,
17575 });
17576 runner.state.save().unwrap();
17577 let newer = git::git(
17580 &runner.state.repo,
17581 &[
17582 "commit-tree",
17583 "main^{tree}",
17584 "-p",
17585 "main",
17586 "-m",
17587 "base advanced during wait",
17588 ],
17589 )
17590 .await
17591 .unwrap();
17592 git::git(
17593 &runner.state.repo,
17594 &[
17595 "push",
17596 "origin",
17597 &format!("{}:refs/heads/main", newer.trim()),
17598 ],
17599 )
17600 .await
17601 .unwrap();
17602 runner.state = RunState::load(&runner.state.id).unwrap();
17603 runner.state.resource_block = None;
17604 runner.sync_to_base().await.unwrap();
17605 assert!(!runner.state.base_sync_blocked());
17606 let op = &runner.state.base_sync_recovery.operations[0];
17607 assert_eq!(op.head, head);
17608 assert_eq!(op.tip, tip);
17609 assert!(op.finished);
17610 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17611 assert_eq!(runner.state.rebase_fixes.len(), 1);
17612 let latest = git::rev_parse(&runner.state.repo, "origin/main")
17613 .await
17614 .unwrap();
17615 assert_ne!(latest, tip);
17616 runner.state = RunState::load(&runner.state.id).unwrap();
17617 runner.sync_to_base().await.unwrap();
17618 assert!(!runner.state.base_sync_blocked());
17619 assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17620 assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17621 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17622 }
17623
17624 #[cfg(unix)]
17625 #[tokio::test]
17626 async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17627 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17628 runner.state.reserve_base_operation(&head, &tip);
17629 runner.state.save().unwrap();
17630 std::fs::write(
17631 scratch.join("retained"),
17632 "upstream and retained resolution\n",
17633 )
17634 .unwrap();
17635 git::git(&scratch, &["add", "retained"]).await.unwrap();
17636 git::git(
17637 &scratch,
17638 &["-c", "core.editor=true", "rebase", "--continue"],
17639 )
17640 .await
17641 .unwrap();
17642 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17643 assert!(
17644 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17645 .await
17646 .unwrap()
17647 .is_empty()
17648 );
17649 assert_eq!(
17650 crate::rebase::finish(
17651 &mut runner.state,
17652 &scratch,
17653 "fix/retained",
17654 &head,
17655 &tip,
17656 &["retained".into()],
17657 ""
17658 )
17659 .await
17660 .unwrap(),
17661 crate::rebase::Rebased::Applied
17662 );
17663 assert!(!scratch.exists());
17664 runner.state = RunState::load(&runner.state.id).unwrap();
17666 let op = &runner.state.base_sync_recovery.operations[0];
17667 assert!(!op.finished);
17668 assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17669 runner.sync_to_base().await.unwrap();
17670 assert!(!runner.state.base_sync_blocked());
17671 assert!(runner.state.base_sync_recovery.operations[0].finished);
17672 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17673 assert_eq!(runner.state.rebase_fixes.len(), 1);
17674 let winner = runner.state.winner().unwrap().clone();
17675 let wait = runner.repository_wait();
17676 assert!(
17677 verify_current_head(&mut runner.state, &winner, &wait)
17678 .await
17679 .unwrap()
17680 );
17681 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17682 }
17683
17684 #[cfg(unix)]
17685 #[tokio::test]
17686 async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17687 for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17688 let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17689 runner.state.reserve_base_operation(&head, &tip);
17690 runner.state.save().unwrap();
17691 std::fs::write(
17692 scratch.join("retained"),
17693 "upstream and retained resolution\n",
17694 )
17695 .unwrap();
17696 git::git(&scratch, &["add", "retained"]).await.unwrap();
17697 git::git(
17698 &scratch,
17699 &["-c", "core.editor=true", "rebase", "--continue"],
17700 )
17701 .await
17702 .unwrap();
17703 let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17704 assert!(!git::rebase_in_progress(&scratch).await);
17705 assert!(
17706 !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17707 .await
17708 .unwrap()
17709 .is_empty()
17710 );
17711 match damage {
17712 Some("record") => {
17713 let path =
17714 git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17715 .await
17716 .unwrap();
17717 std::fs::write(scratch.join(path), "").unwrap();
17718 }
17719 Some("foreign") => {
17720 std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17721 git::git(&scratch, &["add", "foreign"]).await.unwrap();
17722 git::git(&scratch, &["commit", "-m", "foreign work"])
17723 .await
17724 .unwrap();
17725 }
17726 Some("dirty") => {
17727 std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17728 }
17729 _ => {}
17730 }
17731 let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17732 runner.state = RunState::load(&runner.state.id).unwrap();
17734 assert!(
17735 runner.state.base_sync_recovery.operations[0]
17736 .validated_head
17737 .is_none()
17738 );
17739 runner.sync_to_base().await.unwrap();
17740 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17741 assert_eq!(runner.state.base_episode_used(), 1);
17742 assert_eq!(runner.state.rebase_fixes.len(), 1);
17743 assert_eq!(
17744 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17745 retained_head
17746 );
17747 if damage.is_some() {
17748 assert!(runner.state.base_sync_blocked());
17749 assert!(scratch.exists());
17750 assert!(!runner.state.base_sync_recovery.operations[0].finished);
17751 assert!(
17752 runner.state.base_sync_recovery.operations[0]
17753 .validated_head
17754 .is_none()
17755 );
17756 } else {
17757 assert!(!runner.state.base_sync_blocked());
17758 assert!(!scratch.exists());
17759 assert!(runner.state.base_sync_recovery.operations[0].finished);
17760 assert_eq!(
17761 runner.state.base_sync_recovery.operations[0]
17762 .validated_head
17763 .as_deref(),
17764 Some(result.as_str())
17765 );
17766 let winner = runner.state.winner().unwrap().clone();
17767 let wait = runner.repository_wait();
17768 assert!(
17769 verify_current_head(&mut runner.state, &winner, &wait)
17770 .await
17771 .unwrap()
17772 );
17773 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17774 runner.state = RunState::load(&runner.state.id).unwrap();
17775 runner.sync_to_base().await.unwrap();
17776 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17777 }
17778 }
17779 }
17780
17781 #[cfg(unix)]
17782 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17783 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17784 runner.state.prep_complete = Some(true);
17786 runner.state.base_sync = None;
17787 runner.state.base_sync_recovery = Default::default();
17788 runner.state.config.graph.gate_fix_rounds = 1;
17789 let mut review = review_round(true, 0, 1, 1, false, true);
17790 review.head = head.clone();
17791 review.verified_head = Some(head.clone());
17792 runner.state.reviews = vec![review];
17793 runner.state.gate = vec![CommandOutcome {
17794 command: "old gate".into(),
17795 code: Some(1),
17796 output_tail: "old failure".into(),
17797 duration_ms: 0,
17798 resource_blocked: false,
17799 }];
17800 runner.state.gate_head = Some(head.clone());
17801 runner.state.gate_ran = true;
17802 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17803 agent: "fixture".into(),
17804 failed: runner.state.gate.clone(),
17805 notes: "spent".into(),
17806 committed: false,
17807 error: None,
17808 }];
17809 runner.state.land_approval = Some(crate::run::LandApproval {
17810 question: "old approval".into(),
17811 head: head.clone(),
17812 });
17813 runner.state.status = RunStatus::Blocked;
17814 (tmp, runner, head)
17815 }
17816
17817 #[cfg(unix)]
17818 #[tokio::test]
17819 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17820 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17821 let (tmp, mut runner, old) = resumed_gate_fixture().await;
17822 let phases = tmp.path().join("phases");
17823 runner.state.config.verify.e2e = vec![format!(
17824 "echo e2e >> '{}'; {}",
17825 phases.display(),
17826 if e2e_ok {
17827 "true"
17828 } else {
17829 "echo new-e2e-failure; false"
17830 }
17831 )];
17832 runner.state.config.verify.gate = vec![format!(
17833 "echo gate >> '{}'; {}",
17834 phases.display(),
17835 if gate_ok {
17836 "true"
17837 } else {
17838 "echo new-gate-failure; false"
17839 }
17840 )];
17841 runner.state.save().unwrap();
17842 runner.state = RunState::load(&runner.state.id).unwrap();
17843 runner.sync_to_base().await.unwrap();
17844 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17845 assert_ne!(head, old);
17846 runner.state = RunState::load(&runner.state.id).unwrap();
17848 runner.execute_graph().await.unwrap();
17849 assert_eq!(runner.state.gate_fixes.len(), 1);
17850 assert_eq!(runner.state.reviews.len(), 1);
17851 assert_eq!(runner.state.reviews[0].head, old);
17852 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17853 assert_eq!(runner.state.verification_history.len(), 1);
17854 let history = &runner.state.verification_history[0];
17855 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17856 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17857 assert!(history.archived_at.is_some());
17858 assert_eq!(history.fixes_spent, Some(1));
17859 assert_eq!(history.gate[0].output_tail, "old failure");
17860 let archive_events: Vec<_> = runner
17861 .state
17862 .events
17863 .iter()
17864 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17865 .collect();
17866 assert_eq!(archive_events.len(), 1);
17867 assert!(archive_events[0].message.contains(&short(&old)));
17868 assert!(archive_events[0].message.contains(&short(&head)));
17869 assert!(
17870 archive_events[0]
17871 .message
17872 .contains("1 gate-fix round(s) already spent stay spent")
17873 );
17874 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17875 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17876 if e2e_ok {
17877 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17878 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17879 } else {
17880 assert!(!runner.state.gate_ran);
17881 }
17882 assert_eq!(
17883 runner.state.status,
17884 if e2e_ok && gate_ok {
17885 RunStatus::Ready
17886 } else {
17887 RunStatus::Blocked
17888 }
17889 );
17890 if !e2e_ok || !gate_ok {
17891 let cause = runner.state.verification_blocking_cause().unwrap();
17892 assert!(cause.contains(&head));
17893 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17894 runner.state.event("land", "rebased successfully");
17896 let notice = crate::notices::run_ended(&runner.state).unwrap();
17897 assert!(!format!("{notice:?}").contains("rebased successfully"));
17898 assert!(runner.state.merge.is_none());
17899 }
17900 let before = std::fs::read_to_string(&phases).unwrap();
17901 runner.state.save().unwrap();
17902 runner.state = RunState::load(&runner.state.id).unwrap();
17903 runner.review_loop().await.unwrap();
17904 runner.gate().await.unwrap();
17905 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17906 assert_eq!(runner.state.verification_history.len(), 1);
17907 let history = &runner.state.verification_history[0];
17908 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17909 assert!(history.archived_at.is_some());
17910 assert_eq!(history.fixes_spent, Some(1));
17911 }
17912 }
17913
17914 #[cfg(unix)]
17915 #[tokio::test]
17916 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17917 let (tmp, mut runner, head) = resumed_gate_fixture().await;
17918 let marker = tmp.path().join("gate-ran");
17919 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17920 runner.review_loop().await.unwrap();
17921 runner.gate().await.unwrap();
17922 assert_eq!(runner.state.status, RunStatus::Blocked);
17923 assert!(!marker.exists());
17924 assert!(runner.state.verification_history.is_empty());
17925 runner.state.gate_head = None;
17926 runner.gate().await.unwrap();
17927 assert!(marker.exists());
17928 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17929 assert_eq!(runner.state.verification_history.len(), 1);
17930 assert_eq!(runner.state.gate_fixes.len(), 1);
17931 }
17932
17933 #[cfg(unix)]
17934 #[tokio::test]
17935 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17936 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17937 runner.sync_to_base().await.unwrap();
17938 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17939 runner.state.reconcile_verification_head(&head);
17940 for outcomes in [
17942 Vec::new(),
17943 vec![CommandOutcome {
17944 command: "true".into(),
17945 code: None,
17946 output_tail: "cache unavailable".into(),
17947 duration_ms: 0,
17948 resource_blocked: true,
17949 }],
17950 ] {
17951 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17952 head: head.clone(),
17953 passed: false,
17954 outcomes,
17955 });
17956 runner.state.gate_ran = false;
17957 runner.state.gate.clear();
17958 runner.state.save().unwrap();
17959 runner.state = RunState::load(&runner.state.id).unwrap();
17960 runner.merge().await.unwrap();
17961 assert!(runner.state.merge.is_none());
17962 runner.review_loop().await.unwrap();
17963 runner.gate().await.unwrap();
17964 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17965 assert!(runner.state.gate_allows_progress());
17966 assert_eq!(runner.state.gate_fixes.len(), 1);
17967 assert!(runner.state.merge.is_none());
17968 }
17969 git::git(
17971 &runner.state.repo,
17972 &["commit", "--allow-empty", "-m", "external update"],
17973 )
17974 .await
17975 .unwrap();
17976 runner.merge().await.unwrap();
17977 assert!(runner.state.merge.is_none());
17978 assert_eq!(runner.state.status, RunStatus::Blocked);
17979 }
17980
17981 #[cfg(unix)]
17982 #[tokio::test]
17983 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17984 for waiting_on_gate in [false, true] {
17985 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17986 runner.sync_to_base().await.unwrap();
17987 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17988 runner.state.reconcile_verification_head(&head);
17989 if waiting_on_gate {
17990 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17991 head: head.clone(),
17992 passed: true,
17993 outcomes: Vec::new(),
17994 });
17995 }
17996 let cache = tmp.path().join("cache");
17997 runner.state.config.verify.e2e =
17998 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17999 runner.state.config.verify.gate = vec!["true".into()];
18000 runner.state.config.graph.timeout_resource_wait = 1;
18001 let owner =
18002 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18003 let guard =
18004 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18005 crate::cache::AcquireOutcome::Acquired(g) => g,
18006 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18007 };
18008 runner.review_loop().await.unwrap();
18009 runner.gate().await.unwrap();
18010 assert!(!runner.state.gate_ran);
18011 assert_eq!(runner.state.gate_fixes.len(), 1);
18012 assert!(runner.state.resource_block.is_some());
18013 runner.merge().await.unwrap();
18014 assert!(runner.state.merge.is_none());
18015 drop(guard);
18016 runner.state = RunState::load(&runner.state.id).unwrap();
18017 runner.state.resource_block = None;
18019 runner.review_loop().await.unwrap();
18020 runner.gate().await.unwrap();
18021 assert!(runner.state.gate_allows_progress());
18022 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18023 assert_eq!(runner.state.gate_fixes.len(), 1);
18024 }
18025 }
18026
18027 #[cfg(unix)]
18028 #[tokio::test]
18029 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18030 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18031 runner.sync_to_base().await.unwrap();
18032 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18033 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18034 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18036 runner.review_loop().await.unwrap();
18037 runner.gate().await.unwrap();
18038 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18039 assert_ne!(before, after);
18040 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18041 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18042 assert!(!runner.state.gate_allows_progress());
18043 runner.merge().await.unwrap();
18044 assert!(runner.state.merge.is_none());
18045 assert!(
18046 runner
18047 .state
18048 .verification_history
18049 .iter()
18050 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18051 );
18052
18053 runner.state.reviews[0].clean = false;
18055 runner.state.reviews[0].answered = 0;
18056 runner.state.config.verify.e2e = vec!["true".into()];
18057 runner.review_loop().await.unwrap();
18058 runner.gate().await.unwrap();
18059 runner.merge().await.unwrap();
18060 assert_eq!(runner.state.status, RunStatus::Blocked);
18061 assert!(runner.state.merge.is_none());
18062 assert!(
18063 runner
18064 .state
18065 .verification_blocking_cause()
18066 .unwrap()
18067 .contains("review blocked")
18068 );
18069 }
18070
18071 #[cfg(unix)]
18072 #[tokio::test]
18073 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18074 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18075 runner.state.gate_head = None;
18076 runner.state.config.verify.gate =
18077 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18078 runner.review_loop().await.unwrap();
18079 runner.gate().await.unwrap();
18080 assert_ne!(
18081 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18082 old
18083 );
18084 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18085 assert!(!runner.state.gate_allows_progress());
18086 assert_eq!(runner.state.status, RunStatus::Blocked);
18087 runner.merge().await.unwrap();
18088 assert!(runner.state.merge.is_none());
18089 }
18090
18091 #[cfg(unix)]
18092 #[tokio::test]
18093 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18094 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18095 let mut old_review = review_round(true, 0, 1, 1, false, true);
18096 old_review.head = head.clone();
18097 old_review.verified_head = Some(head.clone());
18098 runner.state.reviews = vec![old_review];
18099 runner.state.gate = vec![CommandOutcome {
18100 command: "old gate".into(),
18101 code: None,
18102 output_tail: "timed out after 2400s".into(),
18103 duration_ms: 2_400_000,
18104 resource_blocked: false,
18105 }];
18106 runner.state.gate_ran = true;
18107 runner.state.gate_head = Some(head.clone());
18108 runner.state.land_approval = Some(crate::run::LandApproval {
18109 question: "old-head-approval".into(),
18110 head: head.clone(),
18111 });
18112 runner.state.authorize_base_sync(&task).unwrap();
18113 runner.sync_to_base().await.unwrap();
18114 assert!(!runner.state.base_sync_blocked());
18115 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18116 assert_ne!(head, fresh);
18117 assert_eq!(runner.state.reviews.len(), 1);
18118 assert!(!runner.state.gate_ran);
18119 let saved = &runner.state.base_sync_recovery.authorizations[0];
18120 assert_eq!(saved.reviews[0].head, head);
18121 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18122 assert!(saved.gate_ran);
18123 assert!(!saved.gate[0].ok());
18124 runner.review_loop().await.unwrap();
18125 runner.gate().await.unwrap();
18126 assert_eq!(runner.state.reviews[0].head, head);
18127 assert_eq!(
18128 runner.state.reviews[0].verified_head.as_deref(),
18129 Some(head.as_str())
18130 );
18131 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18132 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18133 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18134 assert!(runner.state.gate_ran);
18135 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18136 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18137 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18138 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18139 assert_eq!(
18140 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18141 "retained change"
18142 );
18143 }
18144
18145 #[cfg(unix)]
18146 #[tokio::test]
18147 async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18148 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18149 let queue = crate::queue::Queue::open();
18150 let mut owner = queue.get(&task).unwrap();
18151 owner.release_operator("test-resume", None);
18152 queue.put(&mut owner).unwrap();
18153 let generation = owner.resumption.as_ref().unwrap().generation;
18154 runner.state.task_instruction = Some(owner.instruction.clone());
18155 runner.state.driver_exited = true;
18156 runner.state.save().unwrap();
18157 let home = crate::run::home();
18158 let _claim = queue.claim(&task).unwrap();
18159 owner.resumption.as_mut().unwrap().at =
18160 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18161 queue.put(&mut owner).unwrap();
18162 assert!(
18163 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18164 .await
18165 .is_err(),
18166 "an approval from before the stop grants nothing"
18167 );
18168 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18169 queue.put(&mut owner).unwrap();
18170 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18171 .await
18172 .unwrap();
18173 assert!(
18174 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18175 .await
18176 .is_err()
18177 );
18178 runner.state = RunState::load(&runner.state.id).unwrap();
18179 assert_eq!(
18180 runner.state.base_sync_recovery.authorizations[0].generation,
18181 Some(generation)
18182 );
18183 runner.sync_to_base().await.unwrap();
18184 let repo = runner.state.repo.clone();
18185 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18186 git::git(&repo, &["checkout", "main"]).await.unwrap();
18187 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18188 git::git(&repo, &["add", "."]).await.unwrap();
18189 git::git(&repo, &["commit", "-m", "move base again"])
18190 .await
18191 .unwrap();
18192 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18193 git::git(&repo, &["checkout", "fix/retained"])
18194 .await
18195 .unwrap();
18196 runner.sync_to_base().await.unwrap();
18197 assert!(!runner.state.base_sync_blocked());
18198 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18199 assert_eq!(runner.state.base_episode_used(), 2);
18200 let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18201 assert_ne!(fresh, head);
18202 assert!(!runner.state.base_recovery_pending());
18203 owner.resumption.as_mut().unwrap().at = Timestamp::now();
18205 queue.put(&mut owner).unwrap();
18206 assert!(
18207 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18208 .await
18209 .is_err()
18210 );
18211 runner.state = RunState::load(&runner.state.id).unwrap();
18212 runner.sync_to_base().await.unwrap();
18213 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18214 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18215 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18216 }
18217
18218 #[cfg(unix)]
18219 #[tokio::test]
18220 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18221 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18222 runner.state.base_sync = None;
18223 runner.state.status = RunStatus::Blocked;
18224 let mut second = runner.state.candidates[0].clone();
18225 second.label = 'B';
18226 runner.state.candidates.push(second);
18227 assert!(runner.state.votes.is_empty());
18228 assert!(runner.state.deliberation.is_empty());
18229 runner.deliberate().await.unwrap();
18230 runner.vote().await.unwrap();
18231 assert_eq!(runner.state.status, RunStatus::Blocked);
18232 assert!(runner.state.votes.is_empty());
18233 assert!(runner.state.deliberation.is_empty());
18234 }
18235
18236 fn runner_at(status: RunStatus) -> Runner {
18239 let mut state = RunState::new(
18240 PathBuf::from("/nonexistent/repo"),
18241 "main".to_owned(),
18242 "deadbeef".to_owned(),
18243 "task".to_owned(),
18244 Config::default(),
18245 );
18246 state.config.disk.min_free_bytes = 0;
18248 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18251 state.id = format!(
18252 "20000101-000000-{:04x}",
18253 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18254 );
18255 state.status = status;
18256 Runner {
18257 state,
18258 roles: ResolvedRoles {
18259 implementers: Vec::new(),
18260 judges: Vec::new(),
18261 reviewers: Vec::new(),
18262 fixer: None,
18263 conductor: conductor(),
18264 implementer_roster: Vec::new(),
18265 judge_roster: Vec::new(),
18266 reviewer_roster: Vec::new(),
18267 },
18268 sem: Arc::new(Semaphore::new(1)),
18269 pause: Pause::new(),
18270 interrupt: Pause::new(),
18271 }
18272 }
18273
18274 #[tokio::test]
18275 async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18276 crate::run::pin_test_home();
18277 let temp = tempfile::tempdir().unwrap();
18278 let root = temp.path().canonicalize().unwrap();
18279 let repo = root.join("repo");
18280 std::fs::create_dir(&repo).unwrap();
18281 init_repo(&repo);
18282 git::git(&repo, &["config", "core.autocrlf", "false"])
18283 .await
18284 .unwrap();
18285 let mut runner = runner_at(RunStatus::Blocked);
18286 runner.state.repo = repo.clone();
18287 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18288 runner.state.config.agents = vec![conductor()];
18289 runner.state.config.graph.selected_implementer = Some("conductor".into());
18290 runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18291 let (shell, command, setup_output) = if cfg!(windows) {
18292 (
18293 vec!["cmd".into(), "/C".into()],
18294 "echo ready>seat-ready",
18295 "ready\r\n",
18296 )
18297 } else {
18298 (
18299 vec!["sh".into(), "-c".into()],
18300 "printf ready > seat-ready",
18301 "ready",
18302 )
18303 };
18304 runner.state.config.verify.shell = Some(shell);
18305 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18306 run: Some(command.into()),
18307 ..Default::default()
18308 }];
18309 runner.state.save().unwrap();
18310 let path = runner.state.worktree_root().join("judge-1");
18311 git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18312 .await
18313 .unwrap();
18314 std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18315 git::git(&path, &["add", "."]).await.unwrap();
18316 let tree = git::git(&path, &["write-tree"]).await.unwrap();
18317 let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18318 .await
18319 .unwrap();
18320 git::git(&path, &["checkout", "--detach", &head])
18321 .await
18322 .unwrap();
18323 let evidence = crate::fold_evidence::snapshot(
18324 &runner.state,
18325 &crate::run::home().canonicalize().unwrap(),
18326 &head,
18327 64 * 1024 * 1024,
18328 )
18329 .await
18330 .unwrap();
18331 let mut contents = BTreeMap::new();
18332 contents.insert(
18333 repo.join("logs/HEAD"),
18334 format!("{head} {head} fixture\n").into_bytes(),
18335 );
18336 let record = serde_json::json!({
18337 "version":1, "run":runner.state.id, "source":path, "head":head,
18338 "evidence":evidence, "admin":{}, "admin_contents":contents,
18339 "history":{head.clone():evidence}, "phase":"evicted"
18340 });
18341 std::fs::write(
18342 runner.state.dir().join("artifacts/eviction-judge-1.json"),
18343 serde_json::to_vec(&record).unwrap(),
18344 )
18345 .unwrap();
18346 assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18347 git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18348 .await
18349 .unwrap();
18350 git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18351 assert!(
18352 !git::git_raw(&repo, &["cat-file", "-e", &head])
18353 .await
18354 .unwrap()
18355 .ok()
18356 );
18357 let mut resumed = Runner::resume(&runner.state.id).unwrap();
18358 let error = resumed.execute().await.unwrap_err();
18359 assert!(
18360 error
18361 .to_string()
18362 .contains("selected implementation is held"),
18363 "{error:#}"
18364 );
18365 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18366 assert_eq!(
18367 std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18368 setup_output
18369 );
18370 assert_eq!(
18371 std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18372 "unrelated seat"
18373 );
18374 let error = resumed.execute().await.unwrap_err();
18375 assert!(
18376 error
18377 .to_string()
18378 .contains("selected implementation is held"),
18379 "{error:#}"
18380 );
18381 std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18382 git::git(&path, &["add", "."]).await.unwrap();
18383 let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18384 let next = git::git(
18385 &path,
18386 &[
18387 "commit-tree",
18388 &next_tree,
18389 "-p",
18390 &head,
18391 "-m",
18392 "next review checkpoint",
18393 ],
18394 )
18395 .await
18396 .unwrap();
18397 git::git(&path, &["checkout", "--detach", &next])
18398 .await
18399 .unwrap();
18400 resumed.state.reviews.push(
18401 serde_json::from_value(serde_json::json!({
18402 "round":1, "head":next, "reviews":[]
18403 }))
18404 .unwrap(),
18405 );
18406 resumed.state.save().unwrap();
18407 let error = resumed.execute().await.unwrap_err();
18408 assert!(
18409 error
18410 .to_string()
18411 .contains("selected implementation is held"),
18412 "{error:#}"
18413 );
18414 assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18415 }
18416
18417 #[tokio::test]
18418 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18419 crate::run::pin_test_home();
18420 let mut runner = runner_at(RunStatus::Gating);
18421 runner.state.save().unwrap();
18422 let id = runner.state.id.clone();
18423 let passes = runner.state.passes.len();
18424 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18425 let mut claiming = Box::pin(runner.claim_driver());
18426 std::future::poll_fn(|cx| {
18427 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18428 std::task::Poll::Ready(())
18429 })
18430 .await;
18431 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18432 drop(owner);
18433 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18434 .await
18435 .unwrap()
18436 .unwrap();
18437 assert_eq!(execution.runner.state.passes.len(), passes + 1);
18438 assert!(execution.runner.state.resource_block.is_none());
18439 assert_eq!(
18440 execution.runner.state.liveness(false),
18441 crate::run::Liveness::Live
18442 );
18443 drop(execution);
18444 assert!(RunState::load(&id).unwrap().driver_exited);
18445 }
18446
18447 #[tokio::test]
18448 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18449 crate::run::pin_test_home();
18450 let mut runner = runner_at(RunStatus::Gating);
18451 runner.state.save().unwrap();
18452 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18455 assert!(runner.claim_driver().await.is_err());
18456 assert!(runner.state.driver_exited);
18457 assert_eq!(runner.state.driver_pid, None);
18458 let disk = RunState::load(&runner.state.id).unwrap();
18459 assert!(disk.driver_exited);
18460 assert_eq!(disk.driver_pid, None);
18461 assert_eq!(disk.status, RunStatus::Gating);
18462 }
18463
18464 #[tokio::test]
18465 async fn an_early_graph_error_releases_its_driver() {
18466 crate::run::pin_test_home();
18467 let mut runner = runner_at(RunStatus::Blocked);
18468 runner.state.config.graph.selected_implementer = Some("selected".into());
18469 runner.state.save().unwrap();
18470 assert!(runner.execute().await.is_err());
18471 let disk = RunState::load(&runner.state.id).unwrap();
18472 assert!(disk.driver_exited);
18473 assert_eq!(disk.driver_pid, None);
18474 assert_eq!(disk.status, RunStatus::Blocked);
18475 }
18476
18477 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18478 crate::run::MergeRepair {
18479 question: "repair-question".into(),
18480 head: "old-head".into(),
18481 pr: "https://github.com/o/r/pull/42".into(),
18482 reviews: vec![review_round(false, 1, 1, 1, false, true)],
18483 gate: vec![outcome(Some(0), false)],
18484 gate_head: Some("old-head".into()),
18485 gate_fixes: Vec::new(),
18486 pre_gate: Vec::new(),
18487 phase,
18488 failure: None,
18489 }
18490 }
18491
18492 #[tokio::test]
18493 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18494 crate::run::pin_test_home();
18495 let mut runner = runner_at(RunStatus::Landing);
18496 runner.state.merge_repairs.push(merge_repair_request(
18497 crate::run::MergeRepairPhase::Requested,
18498 ));
18499 runner.state.save().unwrap();
18500 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18501 assert!(
18502 runner
18503 .execute()
18504 .await
18505 .unwrap_err()
18506 .to_string()
18507 .contains("already running")
18508 );
18509 assert_eq!(
18510 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18511 crate::run::MergeRepairPhase::Requested
18512 );
18513 drop(claim);
18514 let mut disk = runner.state.clone();
18515 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18516 disk.save().unwrap();
18517 assert!(
18518 runner
18519 .execute()
18520 .await
18521 .unwrap_err()
18522 .to_string()
18523 .contains("decision changed")
18524 );
18525 assert_eq!(
18526 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18527 crate::run::MergeRepairPhase::Complete
18528 );
18529 }
18530
18531 #[test]
18532 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18533 crate::run::pin_test_home();
18534 let mut runner = runner_at(RunStatus::Landing);
18535 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18536 runner.state.reviews = request.reviews.clone();
18537 runner.state.gate = request.gate.clone();
18538 runner.state.gate_ran = true;
18539 runner.state.land_approval = Some(crate::run::LandApproval {
18540 question: "old".into(),
18541 head: "old-head".into(),
18542 });
18543 runner.state.merge_repairs.push(request);
18544 runner.begin_merge_repair_review(0).unwrap();
18545 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18546 assert!(!runner.state.gate_status().ok());
18547 assert!(runner.state.land_approval.is_none());
18548 let disk = RunState::load(&runner.state.id).unwrap();
18549 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18550 assert!(disk.merge_repairs[0].gate[0].ok());
18551 assert_eq!(
18552 disk.merge_repairs[0].phase,
18553 crate::run::MergeRepairPhase::Reviewing
18554 );
18555 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18556 assert!(disk.reviews.is_empty());
18557 }
18558
18559 #[tokio::test]
18560 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18561 crate::run::pin_test_home();
18562 for phase in [
18563 crate::run::MergeRepairPhase::Fixing,
18564 crate::run::MergeRepairPhase::Requested,
18565 ] {
18566 let mut runner = runner_at(RunStatus::Landing);
18567 runner.state.config.graph.land_rounds = 0;
18568 runner.state.merge_repairs.push(merge_repair_request(phase));
18569 runner.state.save().unwrap();
18570 assert!(!runner.repair_merge_review().await.unwrap());
18571 assert_eq!(runner.state.status, RunStatus::Blocked);
18572 assert_eq!(
18573 runner.state.merge_repairs[0].phase,
18574 crate::run::MergeRepairPhase::Failed
18575 );
18576 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18577 assert!(
18578 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18579 "interrupted"
18580 } else {
18581 "budget exhausted"
18582 })
18583 );
18584 assert!(runner.state.merge.is_none());
18585 }
18586 }
18587
18588 #[test]
18592 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18593 crate::run::pin_test_home();
18594 let mut runner = runner_at(RunStatus::Implementing);
18595 let interrupt = Pause::new();
18596 runner.watch_interrupt(interrupt.clone());
18597
18598 interrupt.park_because("task a1b2 asked to run first");
18599
18600 assert!(runner.park_here().expect("park_here"));
18601 assert!(runner.state.parked);
18602 let last = runner.state.events.last().expect("a park event");
18603 assert_eq!(last.node, "park");
18604 assert!(
18605 last.message.contains("task a1b2 asked to run first"),
18606 "expected the interrupt reason in {:?}",
18607 last.message
18608 );
18609 }
18610
18611 #[test]
18614 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18615 crate::run::pin_test_home();
18616 let mut runner = runner_at(RunStatus::Implementing);
18617 runner.state.begin_pass();
18618 let shutdown = Pause::new();
18619 runner.on_pause(shutdown.clone());
18620 let upgrade = ParkCause::Upgrade {
18621 from: "0.121.0".to_owned(),
18622 to: Some("0.121.1".to_owned()),
18623 };
18624 shutdown.park_for(upgrade.clone());
18625 shutdown.park_for(ParkCause::Operator);
18626
18627 assert!(runner.park_here().expect("park_here"));
18628 assert_eq!(
18629 runner.state.passes[0].end,
18630 Some(crate::run::PassEnd::Parked { cause: upgrade })
18631 );
18632 let last = runner.state.events.last().expect("a park event");
18633 assert!(last.message.contains("0.121.0"), "{}", last.message);
18634 }
18635
18636 #[tokio::test]
18637 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18638 crate::run::pin_test_home();
18639 let mut runner = runner_at(RunStatus::Reviewing);
18640 runner.state.save().unwrap();
18641 let mut execution = runner.claim_driver().await.expect("first execution");
18642 assert_eq!(execution.runner.state.passes.len(), 1);
18643 execution
18644 .driver
18645 .finish(&mut execution.runner.state)
18646 .await
18647 .unwrap();
18648 }
18649
18650 #[tokio::test]
18651 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18652 crate::run::pin_test_home();
18653 let mut runner = runner_at(RunStatus::Reviewing);
18654 runner.state.config.agents = vec![conductor()];
18655 runner.state.config.graph.timeout_resource_wait = 0;
18656 runner.state.save().unwrap();
18657 let id = runner.state.id.clone();
18658 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18659 assert!(runner.claim_driver().await.is_err());
18660 drop(owner);
18661 drop(runner);
18662 let mut resumed = Runner::resume(&id).unwrap();
18663 assert_eq!(resumed.state.driver_claimed, Some(false));
18664 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18665 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18666 let mut execution = resumed
18667 .claim_driver()
18668 .await
18669 .expect("retry initial claim after reload");
18670 assert_eq!(execution.runner.state.driver_claimed, Some(true));
18671 assert_eq!(execution.runner.state.passes.len(), 1);
18672 execution
18673 .driver
18674 .finish(&mut execution.runner.state)
18675 .await
18676 .unwrap();
18677 }
18678
18679 #[tokio::test]
18680 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18681 crate::run::pin_test_home();
18682 let mut runner = runner_at(RunStatus::Reviewing);
18683 runner.state.config.agents = vec![conductor()];
18684 runner.state.save().unwrap();
18685 let id = runner.state.id.clone();
18686 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18687 let mut resumed = Runner::resume(&id).unwrap();
18688 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18689 assert!(resumed.claim_driver().await.is_err());
18690 assert!(RunState::load(&id).unwrap().passes.is_empty());
18691 drop(lease);
18692 let mut execution = resumed.claim_driver().await.unwrap();
18693 execution
18694 .driver
18695 .finish(&mut execution.runner.state)
18696 .await
18697 .unwrap();
18698 }
18699
18700 #[tokio::test]
18701 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18702 crate::run::pin_test_home();
18703 let mut runner = runner_at(RunStatus::Reviewing);
18704 let state = &mut runner.state;
18705 state.config.agents = vec![conductor()];
18706 state.driver_claimed = None;
18707 state.driver_pid = Some(std::process::id());
18708 state.driver_started_at = None;
18709 state.begin_pass();
18710 state.save().unwrap();
18711 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18712 std::fs::write(
18713 state.dir().join("driver.json"),
18714 serde_json::to_vec(&serde_json::json!({
18715 "pid": state.driver_pid, "started_at": null,
18716 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18717 }))
18718 .unwrap(),
18719 )
18720 .unwrap();
18721 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18722 drop(lease);
18723 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18724 let mut resumed = Runner::resume(&state.id).unwrap();
18725 assert!(!resumed.state.driver_exited);
18726 let mut execution = resumed
18727 .claim_driver()
18728 .await
18729 .expect("lease proves previous execution ended");
18730 assert_eq!(execution.runner.state.passes.len(), 2);
18731 execution
18732 .driver
18733 .finish(&mut execution.runner.state)
18734 .await
18735 .unwrap();
18736 }
18737
18738 #[tokio::test]
18739 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18740 crate::run::pin_test_home();
18741 let mut runner = runner_at(RunStatus::Reviewing);
18742 let mut json = serde_json::to_value(&runner.state).unwrap();
18743 json.as_object_mut().unwrap().remove("driver_claimed");
18744 runner.state = serde_json::from_value(json).unwrap();
18745 assert_eq!(runner.state.driver_claimed, None);
18746 runner.state.save().unwrap();
18747 let mut execution = runner
18748 .claim_driver()
18749 .await
18750 .expect("the execution lease, not a missing marker, excludes a second driver");
18751 assert_eq!(execution.runner.state.passes.len(), 1);
18752 execution
18753 .driver
18754 .finish(&mut execution.runner.state)
18755 .await
18756 .unwrap();
18757 }
18758
18759 #[tokio::test]
18760 async fn driver_admission_refuses_a_proven_live_driver() {
18761 crate::run::pin_test_home();
18762 let mut runner = runner_at(RunStatus::Reviewing);
18763 let pid = std::process::id();
18764 runner.state.driver_pid = Some(pid);
18765 runner.state.driver_started_at = crate::proc::process_started_at(pid);
18766 assert!(runner.state.driver_started_at.is_some());
18767 runner.state.save().unwrap();
18768 let error = runner
18769 .claim_driver()
18770 .await
18771 .err()
18772 .expect("a live driver must refuse");
18773 assert!(error.to_string().contains("live driver"), "{error:#}");
18774 let saved = RunState::load(&runner.state.id).unwrap();
18775 assert_eq!(saved.driver_pid, Some(pid));
18776 assert_eq!(saved.status, RunStatus::Reviewing);
18777 assert!(saved.passes.is_empty());
18778 }
18779
18780 #[test]
18781 fn parking_resource_halt_persists_the_pass_once() {
18782 crate::run::pin_test_home();
18783 let mut runner = runner_at(RunStatus::Reviewing);
18784 runner.state.begin_pass();
18785 runner.state.resource_block = Some(crate::run::ResourceBlock {
18786 node: "fix".into(),
18787 kind: "shared build cache".into(),
18788 detail: "cancelled".into(),
18789 at: Timestamp::now(),
18790 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18791 });
18792 let cause = ParkCause::Upgrade {
18793 from: "old".into(),
18794 to: Some("new".into()),
18795 };
18796 runner.pause.park_for(cause.clone());
18797 runner.stop_for_resource().unwrap();
18800 assert!(runner.resource_halt().unwrap());
18801 assert!(runner.park_here().unwrap());
18802 let saved = RunState::load(&runner.state.id).unwrap();
18803 assert!(saved.parked);
18804 assert_eq!(saved.status, RunStatus::Reviewing);
18805 assert!(saved.passes[0].ended_at.is_some());
18806 assert_eq!(
18807 saved.passes[0].end,
18808 Some(crate::run::PassEnd::Parked {
18809 cause: cause.clone()
18810 })
18811 );
18812 assert_eq!(
18813 saved.resource_block.unwrap().cause,
18814 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18815 );
18816 assert_eq!(
18817 saved
18818 .events
18819 .iter()
18820 .filter(|event| event.node == "park")
18821 .count(),
18822 1
18823 );
18824 }
18825
18826 #[test]
18834 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18835 crate::run::pin_test_home();
18836 let mut runner = runner_at(RunStatus::Implementing);
18837 let shutdown = Pause::new();
18838 runner.on_pause(shutdown.clone());
18839 let interrupt = Pause::new();
18840 runner.watch_interrupt(interrupt.clone());
18841
18842 assert!(!runner.park_here().expect("park_here"));
18844 assert!(!runner.state.parked);
18845
18846 interrupt.park_because("test");
18848 assert!(!shutdown.parked());
18849 assert!(runner.park_here().expect("park_here"));
18850 }
18851
18852 #[tokio::test]
18866 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18867 crate::run::pin_test_home();
18868 let mut runner = runner_at(RunStatus::Implementing);
18869 let interrupt = Pause::new();
18870 runner.watch_interrupt(interrupt.clone());
18871
18872 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18873 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18874
18875 let node = async move {
18879 started_tx.send(()).expect("send started");
18880 finish_rx.await.expect("recv finish");
18881 "node finished"
18882 };
18883
18884 let interrupter = async move {
18885 started_rx.await.expect("recv started");
18886 interrupt.park_because("higher-priority task waiting");
18888 tokio::task::yield_now().await;
18892 finish_tx.send(()).expect("send finish");
18893 };
18894
18895 let (node_result, ()) = tokio::join!(node, interrupter);
18896 assert_eq!(
18897 node_result, "node finished",
18898 "the in-flight call ran to completion"
18899 );
18900
18901 assert!(runner.park_here().expect("park_here"));
18904 assert!(runner.state.parked);
18905 }
18906
18907 #[test]
18913 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18914 crate::run::pin_test_home();
18915 let mut runner = runner_at(RunStatus::Judging);
18916 runner.state.config.agents = vec![conductor()];
18920 runner.state.candidates = vec![Candidate {
18921 index: 0,
18922 label: 'A',
18923 agent: "alpha".to_owned(),
18924 branch: "magi/x/A".to_owned(),
18925 worktree: PathBuf::from("/nonexistent/worktree"),
18926 summary: "did the thing".to_owned(),
18927 stat: "1 file changed".to_owned(),
18928 files: 1,
18929 commits: 1,
18930 empty: false,
18931 failed: None,
18932 verified_noop: None,
18933 duration_ms: 1234,
18934 folded: false,
18935 }];
18936 let run_id = runner.state.id.clone();
18937
18938 let interrupt = Pause::new();
18939 runner.watch_interrupt(interrupt.clone());
18940 interrupt.park_because("task c3d4 asked to run first");
18941 assert!(runner.park_here().expect("park_here"));
18942
18943 let resumed = Runner::resume(&run_id).expect("resume");
18944 assert_eq!(resumed.state.candidates.len(), 1);
18945 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18946 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18947 assert_eq!(resumed.state.status, runner.state.status);
18948 assert!(
18949 resumed.state.parked,
18950 "still parked until `execute` actually walks the graph again"
18951 );
18952 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18953 }
18954
18955 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18957 let mut q = ask::Question::new(
18958 run.to_owned(),
18959 "implement".to_owned(),
18960 "impl-A".to_owned(),
18961 "Which storage backend should the cache use?".to_owned(),
18962 String::new(),
18963 vec!["SQLite".to_owned(), "Redis".to_owned()],
18964 );
18965 store.put(&mut q).unwrap();
18966 q
18967 }
18968
18969 #[test]
18970 fn a_failed_runs_open_question_is_abandoned() {
18971 ask_test_home();
18972 let store = ask::Questions::open();
18973 let mut runner = runner_at(RunStatus::Failed);
18974 let run = runner.state.id.clone();
18975 let q = ask_open_question(&store, &run);
18976
18977 runner.settle_questions();
18978
18979 let back = store.get(&q.id).unwrap();
18980 assert!(
18981 !back.status.open(),
18982 "the seat that asked died with the run; nobody is left to read an answer"
18983 );
18984 assert!(
18985 back.detail.contains(&run) && back.detail.contains("failed"),
18986 "the reason names what the run became, not just that it is gone: {}",
18987 back.detail
18988 );
18989 }
18990
18991 #[test]
18992 fn a_merged_runs_open_question_is_abandoned_too() {
18993 ask_test_home();
18994 let store = ask::Questions::open();
18995 for status in [RunStatus::Merged, RunStatus::Ready] {
18998 let mut runner = runner_at(status);
18999 let run = runner.state.id.clone();
19000 let q = ask_open_question(&store, &run);
19001
19002 runner.settle_questions();
19003
19004 let back = store.get(&q.id).unwrap();
19005 assert!(
19006 !back.status.open(),
19007 "{status:?} run's question must not outlive the run"
19008 );
19009 }
19010 }
19011
19012 #[test]
19013 fn a_still_resumable_runs_open_question_is_left_alone() {
19014 ask_test_home();
19015 let store = ask::Questions::open();
19016 for status in [RunStatus::Blocked, RunStatus::Stalled] {
19022 let mut runner = runner_at(status);
19023 let run = runner.state.id.clone();
19024 let q = ask_open_question(&store, &run);
19025
19026 runner.settle_questions();
19027
19028 let back = store.get(&q.id).unwrap();
19029 assert!(
19030 back.status.open(),
19031 "{status:?} is still alive; the question must still be waiting"
19032 );
19033 }
19034 }
19035
19036 #[test]
19037 fn settle_questions_never_touches_an_already_answered_question() {
19038 ask_test_home();
19039 let store = ask::Questions::open();
19040 let mut runner = runner_at(RunStatus::Failed);
19041 let run = runner.state.id.clone();
19042 let mut q = ask_open_question(&store, &run);
19043 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19044 .unwrap();
19045 store.put(&mut q).unwrap();
19046
19047 runner.settle_questions();
19052 runner.settle_questions();
19053
19054 let back = store.get(&q.id).unwrap();
19055 assert_eq!(
19056 back.status,
19057 ask::QuestionStatus::Answered,
19058 "a real answer is a decision on record, never overwritten by a sweep"
19059 );
19060 }
19061
19062 #[tokio::test]
19073 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19074 crate::run::pin_test_home();
19075 let tmp = tempfile::tempdir().expect("tempdir");
19076 let repo = tmp.path().join("repo");
19077 std::fs::create_dir_all(&repo).unwrap();
19078 init_repo(&repo);
19079
19080 let mut config = Config::default();
19081 config.graph.worktree_root = Some(tmp.path().join("wt"));
19082
19083 let mut state = RunState::new(
19084 repo.clone(),
19085 "main".to_owned(),
19086 "deadbeef".to_owned(),
19087 "task".to_owned(),
19088 config,
19089 );
19090 let root = state.worktree_root();
19091 let wt_a = root.join("cand-A");
19092 let wt_b = root.join("cand-B");
19093 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19094 .await
19095 .expect("worktree A");
19096 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19097 .await
19098 .expect("worktree B");
19099
19100 state.candidates = vec![
19101 Candidate {
19102 index: 0,
19103 label: 'A',
19104 agent: "alpha".to_owned(),
19105 branch: "magi/x/A".to_owned(),
19106 worktree: wt_a.clone(),
19107 summary: String::new(),
19108 stat: String::new(),
19109 files: 0,
19110 commits: 0,
19111 empty: false,
19112 failed: None,
19113 verified_noop: None,
19114 duration_ms: 0,
19115 folded: false,
19116 },
19117 Candidate {
19118 index: 1,
19119 label: 'B',
19120 agent: "beta".to_owned(),
19121 branch: "magi/x/B".to_owned(),
19122 worktree: wt_b.clone(),
19123 summary: String::new(),
19124 stat: String::new(),
19125 files: 0,
19126 commits: 0,
19127 empty: false,
19128 failed: None,
19129 verified_noop: None,
19130 duration_ms: 0,
19131 folded: false,
19132 },
19133 ];
19134 state.tally = Some(Tally {
19135 first_choice: BTreeMap::from([('A', 1)]),
19136 borda: BTreeMap::new(),
19137 winner: 'A',
19138 rankings: 1,
19139 unanimous_initial: true,
19140 deliberated: false,
19141 changed_votes: 0,
19142 unanimous_final: true,
19143 tie_break: None,
19144 judges: 1,
19145 present: 1,
19146 quorum: 1,
19147 met_quorum: true,
19148 uncontested: None,
19149 });
19150 state.status = RunStatus::Ready;
19151
19152 fold_run(&mut state, false, &crate::run::home())
19153 .await
19154 .expect("fold_run");
19155
19156 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19157 assert!(
19158 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19159 "the unmerged winner's branch survives"
19160 );
19161 assert!(
19162 !state.candidates[0].folded,
19163 "the winner is not marked folded"
19164 );
19165
19166 assert!(!wt_b.exists(), "the loser's worktree is removed");
19167 assert!(
19168 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19169 "the loser's branch is removed"
19170 );
19171 assert!(state.candidates[1].folded, "the loser is marked folded");
19172 }
19173
19174 #[tokio::test]
19175 #[cfg(unix)]
19176 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19177 use std::os::fd::AsRawFd as _;
19178 use std::time::Duration;
19179
19180 let tmp = tempfile::tempdir().unwrap();
19181 let repo = tmp.path().join("repo");
19182 std::fs::create_dir_all(&repo).unwrap();
19183 init_repo(&repo);
19184 let common = repo.join(".git");
19185 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19186 let home = tmp.path().join("home");
19187 let mut config = Config::default();
19188 config.coordination = crate::config::test_coordination();
19189 config.graph.worktree_root = Some(tmp.path().join("wt"));
19190 config.graph.timeout_resource_wait = 1;
19191 let mut state = RunState::new(
19192 repo.clone(),
19193 "main".into(),
19194 "deadbeef".into(),
19195 "task".into(),
19196 config,
19197 );
19198 let root = state.worktree_root();
19199 let candidate = root.join("cand-A");
19200 let reviewer = root.join("review-1");
19201 let branch = state.branch_for('A');
19202 git::worktree_add_branch(&repo, &candidate, &branch, "main")
19203 .await
19204 .unwrap();
19205 git::worktree_add_detached(&repo, &reviewer, "main")
19206 .await
19207 .unwrap();
19208 git::acquire_worktree_config(&repo).await.unwrap();
19209 state.enabled_worktree_config = true;
19210 state.status = RunStatus::Ready;
19211 state.candidates.push(Candidate {
19212 index: 0,
19213 label: 'A',
19214 agent: "implement".into(),
19215 branch: branch.clone(),
19216 worktree: candidate.clone(),
19217 summary: String::new(),
19218 stat: String::new(),
19219 files: 0,
19220 commits: 0,
19221 empty: false,
19222 failed: None,
19223 verified_noop: None,
19224 duration_ms: 0,
19225 folded: false,
19226 });
19227 state.save_under(&home).unwrap();
19228 let config_before = std::fs::read(common.join("config")).unwrap();
19229 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19230 .await
19231 .unwrap();
19232 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19235 assert_eq!(
19236 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19237 0
19238 );
19239 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19240 assert!(
19241 tokio::time::timeout(Duration::from_millis(200), &mut fold)
19242 .await
19243 .is_err()
19244 );
19245 assert!(candidate.exists());
19246 assert!(reviewer.exists());
19247 assert!(git::branch_exists(&repo, &branch).await.unwrap());
19248 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19249 assert_eq!(
19250 git::git(&repo, &["worktree", "list", "--porcelain"])
19251 .await
19252 .unwrap(),
19253 worktrees_before
19254 );
19255 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19256 .await
19257 .unwrap()
19258 .unwrap_err();
19259 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19260 drop(fold);
19261 assert!(candidate.exists() && reviewer.exists());
19262 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19263 assert_eq!(
19264 state.resource_block.as_ref().unwrap().kind,
19265 "repository coordination"
19266 );
19267 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19268 drop(external);
19269 let mut fold = Box::pin(fold_run(&mut state, true, &home));
19270 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19271 .await
19272 .unwrap()
19273 .unwrap();
19274 drop(fold);
19275 assert!(removed.contains(&branch));
19276 assert!(!root.exists());
19277 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19278 assert!(state.candidates[0].folded);
19279 assert!(!state.enabled_worktree_config);
19280 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19281 let enabled = git::git_raw(
19282 &repo,
19283 &["config", "--local", "--get", "extensions.worktreeConfig"],
19284 )
19285 .await
19286 .unwrap();
19287 assert!(!enabled.ok(), "the last config reference was released");
19288 }
19289
19290 #[tokio::test]
19293 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19294 let tmp = tempfile::tempdir().expect("tempdir");
19295 let repo = tmp.path().join("repo");
19296 std::fs::create_dir_all(&repo).unwrap();
19297 init_repo(&repo);
19298 let home = tmp.path().join("home");
19299
19300 let mut config = Config::default();
19301 config.graph.worktree_root = Some(tmp.path().join("wt"));
19302 let mut state = RunState::new(
19303 repo.clone(),
19304 "main".to_owned(),
19305 "deadbeef".to_owned(),
19306 "task".to_owned(),
19307 config,
19308 );
19309 git::git(&repo, &["branch", "magi/x/A", "main"])
19311 .await
19312 .expect("branch");
19313 state.candidates = vec![Candidate {
19314 index: 0,
19315 label: 'A',
19316 agent: "alpha".to_owned(),
19317 branch: "magi/x/A".to_owned(),
19318 worktree: state.worktree_root().join("cand-A"),
19319 summary: String::new(),
19320 stat: String::new(),
19321 files: 0,
19322 commits: 0,
19323 empty: false,
19324 failed: None,
19325 verified_noop: None,
19326 duration_ms: 0,
19327 folded: true,
19328 }];
19329 state.released_to = Some("20260901-000000-new1".to_owned());
19330 state.released_branches = vec!["magi/x/A".to_owned()];
19331
19332 fold_run(&mut state, true, &home).await.expect("fold_run");
19333 state.status = RunStatus::Merged;
19334 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19335 state.merge = Some(crate::run::MergeOutcome {
19336 mode: MergeMode::Pr,
19337 ok: true,
19338 detail: "confirmed merged".into(),
19339 empty: false,
19340 });
19341 fold_run(&mut state, true, &home)
19342 .await
19343 .expect("merged fold");
19344
19345 assert!(
19346 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19347 "the handed-over branch survives a fold"
19348 );
19349 }
19350
19351 #[tokio::test]
19355 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19356 let tmp = tempfile::tempdir().expect("tempdir");
19357 let repo = tmp.path().join("repo");
19358 std::fs::create_dir_all(&repo).unwrap();
19359 init_repo(&repo);
19360 let run = |args: &[&str]| {
19361 let out = std::process::Command::new("git")
19362 .quiet()
19363 .args(args)
19364 .current_dir(&repo)
19365 .output()
19366 .expect("spawn git");
19367 assert!(out.status.success(), "git {args:?}");
19368 };
19369 run(&["branch", "magi/x/A"]);
19370 run(&["checkout", "-q", "-b", "magi/x/B"]);
19371 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19372 run(&["add", "-A"]);
19373 run(&["commit", "-q", "-m", "work"]);
19374 run(&["checkout", "-q", "main"]);
19375
19376 let probe = RunState::new(
19380 repo.clone(),
19381 "main".to_owned(),
19382 "deadbeef".to_owned(),
19383 "task".to_owned(),
19384 Config::default(),
19385 );
19386 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19387 let bare = tmp.path().join("origin.git");
19388 let out = std::process::Command::new("git")
19389 .quiet()
19390 .args(["init", "-q", "--bare"])
19391 .arg(&bare)
19392 .output()
19393 .expect("spawn git");
19394 assert!(out.status.success(), "git init --bare");
19395 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19396 run(&["push", "-q", "origin", "main"]);
19397
19398 let mut state = RunState::new(
19399 repo.clone(),
19400 "main".to_owned(),
19401 "deadbeef".to_owned(),
19402 "task".to_owned(),
19403 Config::default(),
19404 );
19405 state.seeds = vec![refs::Seed {
19406 token: "magi/27b2/A".to_owned(),
19407 kind: refs::SeedKind::Unresolved,
19408 sha: String::new(),
19409 branch: true,
19410 detail: "no branch or commit named magi/27b2/A".to_owned(),
19411 }];
19412
19413 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19414 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19415 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19416 let detail = empty_candidate_detail(&state, "main");
19417 assert!(detail.starts_with("empty candidate"), "{detail}");
19418 assert!(!detail.contains("magi/27b2/A"), "{detail}");
19419 assert!(detail.contains("--start-from"), "{detail}");
19420 }
19421
19422 #[tokio::test]
19431 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19432 let tmp = tempfile::tempdir().expect("tempdir");
19433 let repo = tmp.path().join("repo");
19434 std::fs::create_dir_all(&repo).unwrap();
19435 init_repo(&repo);
19436
19437 let mut config = Config::default();
19438 config.merge.mode = MergeMode::Local;
19439
19440 let mut state = RunState::new(
19441 repo.clone(),
19442 "main".to_owned(),
19443 "deadbeef".to_owned(),
19444 "task".to_owned(),
19445 config,
19446 );
19447 state.candidates = vec![Candidate {
19448 index: 0,
19449 label: 'A',
19450 agent: "alpha".to_owned(),
19451 branch: "does-not-exist".to_owned(),
19452 worktree: repo.clone(),
19453 summary: String::new(),
19454 stat: String::new(),
19455 files: 0,
19456 commits: 0,
19457 empty: false,
19458 failed: None,
19459 verified_noop: None,
19460 duration_ms: 0,
19461 folded: false,
19462 }];
19463 state.tally = Some(Tally {
19464 first_choice: BTreeMap::from([('A', 1)]),
19465 borda: BTreeMap::new(),
19466 winner: 'A',
19467 rankings: 1,
19468 unanimous_initial: true,
19469 deliberated: false,
19470 changed_votes: 0,
19471 unanimous_final: true,
19472 tie_break: None,
19473 judges: 0,
19474 present: 0,
19475 quorum: 0,
19476 met_quorum: true,
19477 uncontested: Some("only candidate A produced a change".to_owned()),
19478 });
19479 state.reviews = vec![ReviewRound {
19480 round: 1,
19481 head: "deadbeef".to_owned(),
19482 verified_head: None,
19483 verified_at: None,
19484 reviews: Vec::new(),
19485 e2e: Vec::new(),
19486 fix: None,
19487 blocking: 0,
19488 answered: 0,
19489 expected: 0,
19490 clean: true,
19491 verify_retried: false,
19492 e2e_deferred: false,
19493 e2e_defer_reason: None,
19494 progressed: false,
19495 vote_split: false,
19496 reconsideration: Vec::new(),
19497 verdict: None,
19498 }];
19499 state.gate = vec![CommandOutcome {
19500 command: "test".to_owned(),
19501 code: Some(0),
19502 output_tail: String::new(),
19503 duration_ms: 0,
19504 resource_blocked: false,
19505 }];
19506 state.gate_ran = true;
19507 state.status = RunStatus::Ready;
19512 state.merge = Some(MergeOutcome {
19513 mode: MergeMode::Local,
19514 ok: false,
19515 detail: "already concluded".to_owned(),
19516 empty: false,
19517 });
19518
19519 let mut runner = Runner {
19520 state,
19521 roles: ResolvedRoles {
19522 implementers: Vec::new(),
19523 judges: Vec::new(),
19524 reviewers: Vec::new(),
19525 fixer: None,
19526 conductor: conductor(),
19527 implementer_roster: Vec::new(),
19528 judge_roster: Vec::new(),
19529 reviewer_roster: Vec::new(),
19530 },
19531 sem: Arc::new(Semaphore::new(1)),
19532 pause: Pause::new(),
19533 interrupt: Pause::new(),
19534 };
19535
19536 runner.merge().await.expect("merge");
19537
19538 assert_eq!(
19539 runner.state.status,
19540 RunStatus::Ready,
19541 "a concluded run's status must not change on reentry"
19542 );
19543 assert_eq!(
19544 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19545 Some("already concluded"),
19546 "merge must not run again once the node already recorded an outcome"
19547 );
19548 }
19549
19550 #[tokio::test]
19559 async fn merge_refuses_a_gate_that_has_not_actually_run() {
19560 let tmp = tempfile::tempdir().expect("tempdir");
19561 let repo = tmp.path().join("repo");
19562 std::fs::create_dir_all(&repo).unwrap();
19563 init_repo(&repo);
19564
19565 let mut config = Config::default();
19566 config.merge.mode = MergeMode::Local;
19567
19568 let mut state = RunState::new(
19569 repo.clone(),
19570 "main".to_owned(),
19571 "deadbeef".to_owned(),
19572 "task".to_owned(),
19573 config,
19574 );
19575 state.candidates = vec![Candidate {
19576 index: 0,
19577 label: 'A',
19578 agent: "alpha".to_owned(),
19579 branch: "does-not-exist".to_owned(),
19580 worktree: repo.clone(),
19581 summary: String::new(),
19582 stat: String::new(),
19583 files: 0,
19584 commits: 0,
19585 empty: false,
19586 failed: None,
19587 verified_noop: None,
19588 duration_ms: 0,
19589 folded: false,
19590 }];
19591 state.tally = Some(Tally {
19592 first_choice: BTreeMap::from([('A', 1)]),
19593 borda: BTreeMap::new(),
19594 winner: 'A',
19595 rankings: 1,
19596 unanimous_initial: true,
19597 deliberated: false,
19598 changed_votes: 0,
19599 unanimous_final: true,
19600 tie_break: None,
19601 judges: 0,
19602 present: 0,
19603 quorum: 0,
19604 met_quorum: true,
19605 uncontested: Some("only candidate A produced a change".to_owned()),
19606 });
19607 state.reviews = vec![ReviewRound {
19608 round: 1,
19609 head: "deadbeef".to_owned(),
19610 verified_head: None,
19611 verified_at: None,
19612 reviews: Vec::new(),
19613 e2e: Vec::new(),
19614 fix: None,
19615 blocking: 0,
19616 answered: 0,
19617 expected: 0,
19618 clean: true,
19619 verify_retried: false,
19620 e2e_deferred: false,
19621 e2e_defer_reason: None,
19622 progressed: false,
19623 vote_split: false,
19624 reconsideration: Vec::new(),
19625 verdict: None,
19626 }];
19627 state.gate = Vec::new();
19629 state.gate_ran = false;
19630 state.status = RunStatus::Gating;
19631
19632 let mut runner = Runner {
19633 state,
19634 roles: ResolvedRoles {
19635 implementers: Vec::new(),
19636 judges: Vec::new(),
19637 reviewers: Vec::new(),
19638 fixer: None,
19639 conductor: conductor(),
19640 implementer_roster: Vec::new(),
19641 judge_roster: Vec::new(),
19642 reviewer_roster: Vec::new(),
19643 },
19644 sem: Arc::new(Semaphore::new(1)),
19645 pause: Pause::new(),
19646 interrupt: Pause::new(),
19647 };
19648
19649 runner.merge().await.expect("merge");
19650
19651 assert!(
19652 runner.state.merge.is_none(),
19653 "an empty gate must never be read as a passing one: {:?}",
19654 runner.state.merge
19655 );
19656 }
19657
19658 #[tokio::test]
19665 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19666 let tmp = tempfile::tempdir().expect("tempdir");
19667 let repo = tmp.path().join("repo");
19668 std::fs::create_dir_all(&repo).unwrap();
19669 init_repo(&repo);
19670
19671 let config = Config::default();
19673
19674 let mut state = RunState::new(
19675 repo.clone(),
19676 "main".to_owned(),
19677 "deadbeef".to_owned(),
19678 "task".to_owned(),
19679 config,
19680 );
19681 state.candidates = vec![Candidate {
19682 index: 0,
19683 label: 'A',
19684 agent: "alpha".to_owned(),
19685 branch: "does-not-exist".to_owned(),
19686 worktree: repo.clone(),
19687 summary: String::new(),
19688 stat: String::new(),
19689 files: 0,
19690 commits: 0,
19691 empty: false,
19692 failed: None,
19693 verified_noop: None,
19694 duration_ms: 0,
19695 folded: false,
19696 }];
19697 state.tally = Some(Tally {
19698 first_choice: BTreeMap::from([('A', 1)]),
19699 borda: BTreeMap::new(),
19700 winner: 'A',
19701 rankings: 1,
19702 unanimous_initial: true,
19703 deliberated: false,
19704 changed_votes: 0,
19705 unanimous_final: true,
19706 tie_break: None,
19707 judges: 0,
19708 present: 0,
19709 quorum: 0,
19710 met_quorum: true,
19711 uncontested: Some("only candidate A produced a change".to_owned()),
19712 });
19713 state.reviews = vec![ReviewRound {
19714 round: 1,
19715 head: "deadbeef".to_owned(),
19716 verified_head: None,
19717 verified_at: None,
19718 reviews: Vec::new(),
19719 e2e: Vec::new(),
19720 fix: None,
19721 blocking: 0,
19722 answered: 0,
19723 expected: 0,
19724 clean: true,
19725 verify_retried: false,
19726 e2e_deferred: false,
19727 e2e_defer_reason: None,
19728 progressed: false,
19729 vote_split: false,
19730 reconsideration: Vec::new(),
19731 verdict: None,
19732 }];
19733
19734 let mut runner = Runner {
19735 state,
19736 roles: ResolvedRoles {
19737 implementers: Vec::new(),
19738 judges: Vec::new(),
19739 reviewers: Vec::new(),
19740 fixer: None,
19741 conductor: conductor(),
19742 implementer_roster: Vec::new(),
19743 judge_roster: Vec::new(),
19744 reviewer_roster: Vec::new(),
19745 },
19746 sem: Arc::new(Semaphore::new(1)),
19747 pause: Pause::new(),
19748 interrupt: Pause::new(),
19749 };
19750
19751 runner.gate().await.expect("gate");
19752 assert!(
19753 runner.state.gate_ran,
19754 "zero configured commands is still a real attempt, not an unrun gate"
19755 );
19756 assert!(runner.state.gate.is_empty());
19757 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19758 assert_ne!(
19759 runner.state.status,
19760 RunStatus::Blocked,
19761 "a gate with nothing to check must not read as failed"
19762 );
19763
19764 runner.merge().await.expect("merge");
19765 assert_eq!(
19766 runner.state.status,
19767 RunStatus::Ready,
19768 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19769 );
19770 }
19771
19772 #[tokio::test]
19783 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19784 crate::run::pin_test_home();
19785 let home = crate::run::home();
19786
19787 let tmp = tempfile::tempdir().expect("tempdir");
19788 let repo = tmp.path().join("repo");
19789 std::fs::create_dir_all(&repo).unwrap();
19790 init_repo(&repo);
19791 let cache_dir = tmp.path().join("target");
19794
19795 let mut config = Config::default();
19796 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19797 config.graph.timeout_verify = Some(2);
19801 config.graph.timeout_resource_wait = 0;
19802
19803 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19804 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19805 .expect("no io error acquiring directly")
19806 {
19807 crate::cache::AcquireOutcome::Acquired(g) => g,
19808 crate::cache::AcquireOutcome::Busy(b) => {
19809 panic!("expected the direct acquire to win the lease first: {b:?}")
19810 }
19811 };
19812
19813 let mut state = RunState::new(
19814 repo.clone(),
19815 "main".to_owned(),
19816 "deadbeef".to_owned(),
19817 "task".to_owned(),
19818 config,
19819 );
19820 state.candidates = vec![Candidate {
19821 index: 0,
19822 label: 'A',
19823 agent: "alpha".to_owned(),
19824 branch: "does-not-exist".to_owned(),
19825 worktree: repo.clone(),
19826 summary: String::new(),
19827 stat: String::new(),
19828 files: 0,
19829 commits: 0,
19830 empty: false,
19831 failed: None,
19832 verified_noop: None,
19833 duration_ms: 0,
19834 folded: false,
19835 }];
19836 state.tally = Some(Tally {
19837 first_choice: BTreeMap::from([('A', 1)]),
19838 borda: BTreeMap::new(),
19839 winner: 'A',
19840 rankings: 1,
19841 unanimous_initial: true,
19842 deliberated: false,
19843 changed_votes: 0,
19844 unanimous_final: true,
19845 tie_break: None,
19846 judges: 0,
19847 present: 0,
19848 quorum: 0,
19849 met_quorum: true,
19850 uncontested: Some("only candidate A produced a change".to_owned()),
19851 });
19852 state.reviews = vec![ReviewRound {
19853 round: 1,
19854 head: "deadbeef".to_owned(),
19855 verified_head: None,
19856 verified_at: None,
19857 reviews: Vec::new(),
19858 e2e: Vec::new(),
19859 fix: None,
19860 blocking: 0,
19861 answered: 0,
19862 expected: 0,
19863 clean: true,
19864 verify_retried: false,
19865 e2e_deferred: false,
19866 e2e_defer_reason: None,
19867 progressed: false,
19868 vote_split: false,
19869 reconsideration: Vec::new(),
19870 verdict: None,
19871 }];
19872
19873 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19874 state.reviews[0].e2e = vec![CommandOutcome {
19875 command: state.config.verify.e2e[0].clone(),
19876 code: Some(0),
19877 output_tail: String::new(),
19878 duration_ms: 0,
19879 resource_blocked: false,
19880 }];
19881 let mut runner = Runner {
19882 state,
19883 roles: ResolvedRoles {
19884 implementers: Vec::new(),
19885 judges: Vec::new(),
19886 reviewers: Vec::new(),
19887 fixer: None,
19888 conductor: conductor(),
19889 implementer_roster: Vec::new(),
19890 judge_roster: Vec::new(),
19891 reviewer_roster: Vec::new(),
19892 },
19893 sem: Arc::new(Semaphore::new(1)),
19894 pause: Pause::new(),
19895 interrupt: Pause::new(),
19896 };
19897
19898 runner.gate().await.expect("gate");
19899 assert!(runner.state.resource_block.is_none());
19900 assert!(
19901 !runner
19902 .state
19903 .events
19904 .iter()
19905 .any(|event| { event.message.contains("waiting for the shared build cache") }),
19906 "a gate with nothing to run must never request the held cache lease"
19907 );
19908 assert!(
19909 runner.state.gate_ran,
19910 "zero commands is still a real, immediate attempt"
19911 );
19912 assert!(runner.state.gate.is_empty());
19913 assert_ne!(
19914 runner.state.status,
19915 RunStatus::Blocked,
19916 "must not read as resource-blocked on a lease it never asked for"
19917 );
19918 }
19919
19920 #[cfg(unix)]
19923 #[tokio::test]
19924 async fn verification_is_standalone_inside_a_live_worker() {
19925 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19926 if std::env::var_os(CHILD).is_none() {
19927 let home = tempfile::tempdir().unwrap();
19928 let generation = crate::worker::pin_current(home.path()).unwrap();
19929 let output = std::process::Command::new(std::env::current_exe().unwrap())
19930 .args([
19931 "--exact",
19932 "graph::tests::verification_is_standalone_inside_a_live_worker",
19933 "--nocapture",
19934 ])
19935 .env(CHILD, "1")
19936 .env("MAGI_HOME", home.path())
19937 .env("MAGI_WORKER_HOME", home.path())
19938 .env("MAGI_WORKER_ID", "abcdef")
19939 .env("MAGI_WORKER_GENERATION", &generation.id)
19940 .env("MAGI_WORKER_EXE", &generation.executable)
19941 .env("MAGI_RUN", "verification-parent")
19942 .env("MAGI_NODE", "gate")
19943 .env("MAGI_SEAT", "worker-fixture")
19944 .env("MAGI_PROCESS_OWNER", "fixture-owner")
19945 .env("MAGI_NO_AUTOUPDATE", "0")
19946 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19947 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19948 .env(
19949 "VERIFY_EXPECT_TOOLCHAIN",
19950 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19951 )
19952 .env(
19953 "VERIFY_EXPECT_CACHE",
19954 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19955 )
19956 .output()
19957 .unwrap();
19958 assert!(
19959 output.status.success()
19960 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19961 "{}\n{}",
19962 String::from_utf8_lossy(&output.stdout),
19963 String::from_utf8_lossy(&output.stderr)
19964 );
19965 return;
19966 }
19967 let home = crate::run::home();
19968 let directory = home.join("workers/jobs/abcdef");
19969 std::fs::create_dir_all(&directory).unwrap();
19970 let job = crate::worker::Job {
19971 id: "abcdef".into(),
19972 sequence: 1,
19973 kind: "task".into(),
19974 key: "task:verification".into(),
19975 payload: serde_json::json!({}),
19976 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19977 version: Some(env!("CARGO_PKG_VERSION").into()),
19978 status: crate::worker::Status::Running,
19979 result: None,
19980 pid: Some(std::process::id()),
19981 created_at: 0,
19982 finished_at: None,
19983 };
19984 std::fs::write(
19985 directory.join("job.json"),
19986 serde_json::to_vec(&job).unwrap(),
19987 )
19988 .unwrap();
19989 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19990 let assert_worker = || {
19991 assert_eq!(
19992 crate::worker::current_job(&home).unwrap().unwrap().id,
19993 job.id
19994 );
19995 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19996 assert_eq!(
19997 crate::worker::helper_executable(&home).unwrap(),
19998 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19999 );
20000 assert!(
20001 crate::updater::Checker::new(&crate::config::Update {
20002 mode: crate::config::UpdateMode::Notify,
20003 interval: None,
20004 auto_restart: false,
20005 })
20006 .is_none(),
20007 "the driver remains pinned, even while tests are standalone"
20008 );
20009 };
20010 assert_worker();
20011 let repo = home.join("repo");
20012 std::fs::create_dir_all(&repo).unwrap();
20013 init_repo(&repo);
20014 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20017 test -z "${MAGI_WORKER_GENERATION+x}" &&
20018 test -z "${MAGI_WORKER_HOME+x}" &&
20019 test -z "${MAGI_WORKER_EXE+x}" &&
20020 test "$MAGI_HOME/repo" -ef . &&
20021 test "$MAGI_RUN" = verification-parent &&
20022 test "$MAGI_NODE" = gate &&
20023 test "$MAGI_SEAT" = worker-fixture &&
20024 test -n "$MAGI_PROCESS_OWNER" &&
20025 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20026 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20027 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20028 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20029 assert!(
20031 !std::process::Command::new("sh")
20032 .args(["-c", probe])
20033 .current_dir(&repo)
20034 .status()
20035 .unwrap()
20036 .success()
20037 );
20038 let unrelated_home = tempfile::tempdir().unwrap();
20039 assert!(
20040 crate::worker::current_job(unrelated_home.path()).is_err(),
20041 "an inherited worker id cannot be resolved in a unit fixture's home"
20042 );
20043 let mut runner = runner_at(RunStatus::Gating);
20044 runner.state.repo = repo.clone();
20045 runner.state.config.disk.min_free_bytes = 0;
20046 runner.state.config.coordination = crate::config::test_coordination();
20047 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20048 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20049 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20050 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20051 let winner = Candidate {
20052 index: 0,
20053 label: 'A',
20054 agent: "fixture".into(),
20055 branch: "main".into(),
20056 worktree: repo.clone(),
20057 summary: String::new(),
20058 stat: String::new(),
20059 files: 0,
20060 commits: 0,
20061 empty: false,
20062 failed: None,
20063 verified_noop: None,
20064 duration_ms: 0,
20065 folded: false,
20066 };
20067 let (e2e, retried, pids) = run_e2e_with_retry(
20068 &mut runner.state,
20069 &["sh".into(), "-c".into()],
20070 &[format!("{probe} && printf initial")],
20071 &repo,
20072 Duration::from_secs(10),
20073 "initial",
20074 )
20075 .await;
20076 assert!(!retried && pids.is_empty());
20077 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20078 assert_eq!(e2e[0].output_tail, "initial");
20079 let gate = runner.run_gate(&winner).await.unwrap();
20080 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20081 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20084 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20085 passed: false,
20086 outcomes: Vec::new(),
20087 });
20088 runner.state.save().unwrap();
20089 let mut resumed = RunState::load(&runner.state.id).unwrap();
20090 let wait = crate::repo_lock::WaitPolicy::default();
20091 assert!(matches!(
20092 check_gate_repair_e2e(&mut resumed, &winner, &wait)
20093 .await
20094 .unwrap(),
20095 GateFix::Retry
20096 ));
20097 let saved = RunState::load(&resumed.id).unwrap();
20098 let repair = saved.gate_repair_e2e.unwrap();
20099 assert!(repair.passed);
20100 assert_eq!(repair.outcomes.len(), 1);
20101 assert!(repair.outcomes[0].ok());
20102 let landing = run_landing_gate(&mut resumed, &winner, &wait)
20103 .await
20104 .unwrap();
20105 assert_eq!(landing.len(), 1);
20106 assert!(landing[0].ok(), "{landing:?}");
20107 assert_eq!(
20108 std::fs::read_to_string(repo.join("phases")).unwrap(),
20109 "pre\ngate\ne2e\npre\ngate\n"
20110 );
20111 assert_worker();
20112 }
20113
20114 #[cfg(unix)]
20115 #[tokio::test]
20116 async fn verification_keeps_the_panic_and_the_full_output() {
20117 crate::run::pin_test_home();
20118 let tmp = tempfile::tempdir().unwrap();
20119 let mut config = Config::default();
20120 config.disk.min_free_bytes = 0;
20121 let mut state = RunState::new(
20122 tmp.path().to_owned(),
20123 "main".into(),
20124 "base".into(),
20125 "verification evidence".into(),
20126 config,
20127 );
20128 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20129 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
20130 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20131 let mut names = Vec::new();
20132 for _ in 0..2 {
20133 let (outcomes, _) = run_commands(
20134 &mut state,
20135 "gate",
20136 "gate",
20137 0,
20138 &["sh".into(), "-c".into()],
20139 &[script.into()],
20140 tmp.path(),
20141 Duration::from_secs(60),
20142 )
20143 .await;
20144 let tail = &outcomes[0].output_tail;
20145 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20146 assert!(tail.contains("boom assertion"));
20147 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20148 let line = tail.lines().next().unwrap();
20149 names.push(line.to_owned());
20150 }
20151 assert_ne!(names[0], names[1], "the second run must not overwrite");
20152 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20153 assert!(full.len() > 500_000 && full.contains("boom assertion"));
20154 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20155
20156 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20158 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20159 let (outcomes, _) = run_commands(
20160 &mut state,
20161 "gate",
20162 "gate",
20163 0,
20164 &["sh".into(), "-c".into()],
20165 &["echo failure-output; exit 7".into()],
20166 tmp.path(),
20167 Duration::from_secs(10),
20168 )
20169 .await;
20170 assert_eq!(outcomes[0].code, Some(7));
20171 assert!(state.events.iter().any(|e| {
20172 e.node == "gate" && e.message.contains("could not save the full gate output:")
20173 }));
20174 }
20175
20176 #[cfg(unix)]
20177 #[tokio::test]
20178 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20179 crate::run::pin_test_home();
20180 let tmp = tempfile::tempdir().unwrap();
20181 let mut config = Config::default();
20182 config.disk.min_free_bytes = 0;
20183 let mut state = RunState::new(
20184 tmp.path().to_owned(),
20185 "main".into(),
20186 "base".into(),
20187 "verification evidence".into(),
20188 config,
20189 );
20190 let (outcomes, pids) = run_commands(
20191 &mut state,
20192 "gate",
20193 "gate",
20194 1,
20195 &["sh".into(), "-c".into()],
20196 &[
20197 "printf before-timeout; exec sleep 30".into(),
20198 "kill -TERM $$".into(),
20199 "exit 7".into(),
20200 "printf success".into(),
20201 ],
20202 tmp.path(),
20203 Duration::from_secs(1),
20204 )
20205 .await;
20206 wait_for_timed_out_children_to_die(&pids).await;
20207 assert_eq!(
20208 pids.len(),
20209 1,
20210 "unleased timeout retains its death-check PID"
20211 );
20212 assert_eq!(outcomes.len(), 4);
20213 assert_eq!(outcomes[0].code, None);
20214 assert!(outcomes[0].output_tail.contains("before-timeout"));
20215 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20216 assert_eq!(outcomes[1].code, None);
20217 assert!(outcomes[1].output_tail.contains("command terminated:"));
20218 assert!(!outcomes[1].output_tail.contains("timed out"));
20219 assert_eq!(outcomes[2].code, Some(7));
20220 assert!(!outcomes[2].output_tail.contains("timed out"));
20221 assert_eq!(outcomes[3].code, Some(0));
20222 assert_eq!(outcomes[3].output_tail, "success");
20223 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20224 }
20225
20226 #[tokio::test]
20236 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20237 crate::run::pin_test_home();
20238
20239 let tmp = tempfile::tempdir().expect("tempdir");
20240 let repo = tmp.path().join("repo");
20241 std::fs::create_dir_all(&repo).unwrap();
20242 init_repo(&repo);
20243
20244 let mut config = Config::default();
20245 config.disk.min_free_bytes = 0;
20246 config.verify.gate = vec![
20247 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20248 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20249 .to_owned(),
20250 ];
20251
20252 let mut state = RunState::new(
20253 repo.clone(),
20254 "main".to_owned(),
20255 "deadbeef".to_owned(),
20256 "task".to_owned(),
20257 config,
20258 );
20259 let run_id = state.id.clone();
20260 state.candidates = vec![Candidate {
20261 index: 0,
20262 label: 'A',
20263 agent: "alpha".to_owned(),
20264 branch: "does-not-exist".to_owned(),
20265 worktree: repo.clone(),
20266 summary: String::new(),
20267 stat: String::new(),
20268 files: 0,
20269 commits: 0,
20270 empty: false,
20271 failed: None,
20272 verified_noop: None,
20273 duration_ms: 0,
20274 folded: false,
20275 }];
20276 state.tally = Some(Tally {
20277 first_choice: BTreeMap::from([('A', 1)]),
20278 borda: BTreeMap::new(),
20279 winner: 'A',
20280 rankings: 1,
20281 unanimous_initial: true,
20282 deliberated: false,
20283 changed_votes: 0,
20284 unanimous_final: true,
20285 tie_break: None,
20286 judges: 0,
20287 present: 0,
20288 quorum: 0,
20289 met_quorum: true,
20290 uncontested: Some("only candidate A produced a change".to_owned()),
20291 });
20292 state.reviews = vec![ReviewRound {
20293 round: 1,
20294 head: "deadbeef".to_owned(),
20295 verified_head: None,
20296 verified_at: None,
20297 reviews: Vec::new(),
20298 e2e: Vec::new(),
20299 fix: None,
20300 blocking: 0,
20301 answered: 0,
20302 expected: 0,
20303 clean: true,
20304 verify_retried: false,
20305 e2e_deferred: false,
20306 e2e_defer_reason: None,
20307 progressed: false,
20308 vote_split: false,
20309 reconsideration: Vec::new(),
20310 verdict: None,
20311 }];
20312
20313 let mut runner = Runner {
20314 state,
20315 roles: ResolvedRoles {
20316 implementers: Vec::new(),
20317 judges: Vec::new(),
20318 reviewers: Vec::new(),
20319 fixer: None,
20320 conductor: conductor(),
20321 implementer_roster: Vec::new(),
20322 judge_roster: Vec::new(),
20323 reviewer_roster: Vec::new(),
20324 },
20325 sem: Arc::new(Semaphore::new(1)),
20326 pause: Pause::new(),
20327 interrupt: Pause::new(),
20328 };
20329
20330 let started_marker = repo.join("started.marker");
20331 let release_marker = repo.join("release.marker");
20332 let poller = tokio::spawn(async move {
20333 for _ in 0..100 {
20338 if started_marker.exists()
20339 && let Ok(s) = crate::run::RunState::load(&run_id)
20340 && let Some(a) = s.active.get("gate")
20341 {
20342 std::fs::write(&release_marker, b"go").expect("release marker");
20343 return Some(a.clone());
20344 }
20345 tokio::time::sleep(Duration::from_millis(50)).await;
20346 }
20347 None
20348 });
20349
20350 runner.gate().await.expect("gate");
20351 let captured = poller.await.expect("poller task");
20352 let captured = captured.expect(
20353 "the poller never saw a `gate` task entry in run.json while the command was \
20354 still blocked on its own release marker",
20355 );
20356
20357 assert_eq!(captured.task.as_deref(), Some("gate"));
20358 assert_eq!(captured.node, "gate");
20359 assert_eq!(captured.index, Some(1));
20360 assert_eq!(captured.total, Some(1));
20361 assert!(
20362 captured
20363 .command
20364 .as_deref()
20365 .is_some_and(|c| c.contains("started.marker")),
20366 "{captured:?}"
20367 );
20368
20369 assert!(
20370 runner.state.active.is_empty(),
20371 "the entry must be cleared once the command actually finished: {:?}",
20372 runner.state.active
20373 );
20374 assert!(runner.state.gate_ran);
20375 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20376 }
20377
20378 #[tokio::test]
20381 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20382 use crate::verdict::{Finding, ReviewVote, Severity};
20383 crate::run::pin_test_home();
20384 let tmp = tempfile::tempdir().expect("tempdir");
20385 let repo = tmp.path().join("repo");
20386 std::fs::create_dir_all(&repo).unwrap();
20387 init_repo(&repo);
20388
20389 for (severity, vote, expect) in [
20390 (Severity::Major, ReviewVote::Reject, true),
20391 (Severity::Minor, ReviewVote::Reject, false),
20392 (Severity::Major, ReviewVote::Approve, false),
20393 ] {
20394 let mut round = review_round(false, 1, 1, 1, false, true);
20395 round.reviews = vec![ReviewRecord {
20396 reviewer: 1,
20397 agent: "alpha".to_owned(),
20398 summary: String::new(),
20399 findings: vec![Finding {
20400 id: "R1-1-1".to_owned(),
20401 severity,
20402 file: None,
20403 line: None,
20404 title: "t".to_owned(),
20405 detail: String::new(),
20406 }],
20407 vote: Some(vote),
20408 failed: None,
20409 duration_ms: 0,
20410 attempts: 0,
20411 }];
20412 let mut state = RunState::new(
20413 repo.clone(),
20414 "main".to_owned(),
20415 "deadbeef".to_owned(),
20416 "task".to_owned(),
20417 Config::default(),
20418 );
20419 state.reviews = vec![round];
20420 let mut runner = Runner {
20421 state,
20422 roles: ResolvedRoles {
20423 implementers: Vec::new(),
20424 judges: Vec::new(),
20425 reviewers: Vec::new(),
20426 fixer: None,
20427 conductor: conductor(),
20428 implementer_roster: Vec::new(),
20429 judge_roster: Vec::new(),
20430 reviewer_roster: Vec::new(),
20431 },
20432 sem: Arc::new(Semaphore::new(1)),
20433 pause: Pause::new(),
20434 interrupt: Pause::new(),
20435 };
20436 let shell = runner.state.config.shell();
20437 runner
20438 .stop_reviewing("round budget spent", &shell, &repo)
20439 .await
20440 .expect("stop_reviewing");
20441 assert_eq!(runner.state.status, RunStatus::Gating);
20442 assert_eq!(
20443 runner.state.contested_handoff.is_some(),
20444 expect,
20445 "{severity:?} + {vote:?}"
20446 );
20447 }
20448 }
20449
20450 #[tokio::test]
20463 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20464 crate::run::pin_test_home();
20465 let home = crate::run::home();
20466
20467 let tmp = tempfile::tempdir().expect("tempdir");
20468 let repo = tmp.path().join("repo");
20469 std::fs::create_dir_all(&repo).unwrap();
20470 init_repo(&repo);
20471 let head = crate::git::rev_parse(&repo, "HEAD")
20472 .await
20473 .expect("rev-parse");
20474 let cache_dir = tmp.path().join("target");
20477
20478 let mut config = Config::default();
20479 config.verify.e2e = vec![format!(
20480 "CARGO_TARGET_DIR='{}' test -f README.md",
20481 cache_dir.display()
20482 )];
20483 config.graph.review_rounds = 1;
20484 config.graph.timeout_verify = Some(2);
20487 config.graph.timeout_resource_wait = 2;
20488
20489 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20490 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20491 .expect("no io error acquiring directly")
20492 {
20493 crate::cache::AcquireOutcome::Acquired(g) => g,
20494 crate::cache::AcquireOutcome::Busy(b) => {
20495 panic!("expected the direct acquire to win the lease first: {b:?}")
20496 }
20497 };
20498
20499 let mut state = RunState::new(
20500 repo.clone(),
20501 "main".to_owned(),
20502 head.clone(),
20503 "task".to_owned(),
20504 config,
20505 );
20506 state.candidates = vec![Candidate {
20507 index: 0,
20508 label: 'A',
20509 agent: "alpha".to_owned(),
20510 branch: "does-not-exist".to_owned(),
20511 worktree: repo.clone(),
20512 summary: String::new(),
20513 stat: String::new(),
20514 files: 0,
20515 commits: 0,
20516 empty: false,
20517 failed: None,
20518 verified_noop: None,
20519 duration_ms: 0,
20520 folded: false,
20521 }];
20522 state.tally = Some(Tally {
20523 first_choice: BTreeMap::from([('A', 1)]),
20524 borda: BTreeMap::new(),
20525 winner: 'A',
20526 rankings: 1,
20527 unanimous_initial: true,
20528 deliberated: false,
20529 changed_votes: 0,
20530 unanimous_final: true,
20531 tie_break: None,
20532 judges: 0,
20533 present: 0,
20534 quorum: 0,
20535 met_quorum: true,
20536 uncontested: Some("only candidate A produced a change".to_owned()),
20537 });
20538 state.reviews = vec![ReviewRound {
20542 round: 1,
20543 head: head.clone(),
20544 verified_head: None,
20545 verified_at: None,
20546 reviews: Vec::new(),
20547 e2e: Vec::new(),
20548 fix: None,
20549 blocking: 1,
20550 answered: 1,
20551 expected: 1,
20552 clean: false,
20553 verify_retried: false,
20554 e2e_deferred: true,
20555 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20556 progressed: false,
20557 vote_split: false,
20558 reconsideration: Vec::new(),
20559 verdict: None,
20560 }];
20561
20562 let mut runner = Runner {
20563 state,
20564 roles: ResolvedRoles {
20565 implementers: Vec::new(),
20566 judges: Vec::new(),
20567 reviewers: Vec::new(),
20568 fixer: None,
20569 conductor: conductor(),
20570 implementer_roster: Vec::new(),
20571 judge_roster: Vec::new(),
20572 reviewer_roster: Vec::new(),
20573 },
20574 sem: Arc::new(Semaphore::new(1)),
20575 pause: Pause::new(),
20576 interrupt: Pause::new(),
20577 };
20578
20579 let shell = runner.state.config.shell();
20580 runner
20581 .stop_reviewing("round budget spent", &shell, &repo)
20582 .await
20583 .expect("stop_reviewing");
20584
20585 let last = runner.state.reviews.last().expect("round record");
20586 assert_eq!(
20587 last.e2e_status(),
20588 E2eStatus::ResourceBlocked,
20589 "the shared cache is still held; the attempt must read as blocked, not deferred or \
20590 failed: {last:?}"
20591 );
20592 assert_eq!(
20593 last.verified_head.as_deref(),
20594 Some(head.as_str()),
20595 "which commit this attempt targeted is known even though nothing finished checking \
20596 it"
20597 );
20598 let first_attempt_at = last
20599 .verified_at
20600 .expect("when this attempt ran is known too");
20601 assert_ne!(
20602 runner.state.status,
20603 RunStatus::Blocked,
20604 "contention is evidence about the machine, not the patch — it must not settle the \
20605 run as blocked: {:?}",
20606 runner.state.status
20607 );
20608 assert!(
20609 !runner
20610 .state
20611 .events
20612 .iter()
20613 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20614 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20615 runner.state.events
20616 );
20617
20618 runner
20623 .stop_reviewing("round budget spent", &shell, &repo)
20624 .await
20625 .expect("stop_reviewing retry");
20626 assert_eq!(
20627 runner.state.reviews.len(),
20628 1,
20629 "no new round was started: {:?}",
20630 runner.state.reviews
20631 );
20632 let last = runner.state.reviews.last().expect("round record");
20633 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20634 assert!(
20635 last.verified_at.expect("still known") > first_attempt_at,
20636 "a second reentry must be a fresh attempt, not a stale copy of the first"
20637 );
20638 assert_ne!(runner.state.status, RunStatus::Blocked);
20639
20640 held.release();
20641 }
20642
20643 #[cfg(unix)]
20644 #[tokio::test]
20645 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20646 ask_test_home();
20647 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20648 let tmp = tempfile::tempdir().unwrap();
20649 let repo = tmp.path().join("repo");
20650 std::fs::create_dir(&repo).unwrap();
20651 init_repo(&repo);
20652 let run_git = |args: &[&str]| {
20653 let out = std::process::Command::new("git")
20654 .args(args)
20655 .current_dir(&repo)
20656 .output()
20657 .unwrap();
20658 assert!(
20659 out.status.success(),
20660 "{args:?}: {}",
20661 String::from_utf8_lossy(&out.stderr)
20662 );
20663 };
20664 let origin = tmp.path().join("origin.git");
20665 run_git(&["init", "--bare", origin.to_str().unwrap()]);
20666 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20667 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20668 run_git(&["add", "spec.rs"]);
20669 run_git(&["commit", "-m", "initial spec"]);
20670 run_git(&["push", "origin", "main"]);
20671 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20672 run_git(&["checkout", "-b", "fix/review"]);
20673 std::fs::write(
20674 repo.join("check.rs"),
20675 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20676 )
20677 .unwrap();
20678 run_git(&["add", "check.rs"]);
20679 run_git(&["commit", "-m", "use spec"]);
20680 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20681 run_git(&["checkout", "main"]);
20682 std::fs::write(
20683 repo.join("spec.rs"),
20684 "struct AgentSpec { max_turn_secs: u64 }\n",
20685 )
20686 .unwrap();
20687 run_git(&["add", "spec.rs"]);
20688 run_git(&["commit", "-m", "extend spec"]);
20689 run_git(&["push", "origin", "main"]);
20690 run_git(&["checkout", "fix/review"]);
20691
20692 let mut runner = runner_at(RunStatus::Reviewing);
20693 runner.state.repo = repo.clone();
20694 runner.state.base_commit = base;
20695 runner.state.pr_head = Some("fix/review".into());
20696 runner.state.config.graph.review_rounds = max_rounds;
20697 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20698 runner.state.config.graph.sessions = false;
20699 runner.state.config.disk.min_free_bytes = 0;
20700 runner.state.config.verify.e2e =
20701 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20702 runner.state.candidates = vec![Candidate {
20703 index: 0,
20704 label: 'A',
20705 agent: "fixture".into(),
20706 branch: "fix/review".into(),
20707 worktree: repo.clone(),
20708 summary: String::new(),
20709 stat: String::new(),
20710 files: 1,
20711 commits: 1,
20712 empty: false,
20713 failed: None,
20714 verified_noop: None,
20715 duration_ms: 0,
20716 folded: false,
20717 }];
20718 runner.state.tally = Some(Tally {
20719 first_choice: BTreeMap::new(),
20720 borda: BTreeMap::new(),
20721 winner: 'A',
20722 rankings: 0,
20723 unanimous_initial: true,
20724 deliberated: false,
20725 changed_votes: 0,
20726 unanimous_final: true,
20727 tie_break: None,
20728 judges: 0,
20729 present: 0,
20730 quorum: 0,
20731 met_quorum: true,
20732 uncontested: Some("review-only".into()),
20733 });
20734 let mut last = review_round(false, 0, 1, 1, false, false);
20735 last.head = reviewed.clone();
20736 last.e2e_deferred = true;
20737 last.e2e[0].resource_blocked = true;
20738 last.e2e[0].output_tail = "waiting for the shared build cache".into();
20739 if max_rounds == 1 {
20740 last.e2e.clear();
20741 }
20742 runner.state.reviews = vec![last];
20743 let calls = tmp.path().join("fix-calls");
20744 let mut fixer = spec("repair");
20745 fixer
20746 .env
20747 .insert("FIX_CALLS".into(), calls.display().to_string());
20748 fixer.command = vec!["sh".into(), "-c".into(), r#"
20749 echo fix >> "$FIX_CALLS"
20750 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20751 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20752 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20753 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20754 "#.into()];
20755 let mut reviewer = spec("reviewer");
20756 reviewer.command = vec![
20757 "sh".into(),
20758 "-c".into(),
20759 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20760 .into(),
20761 ];
20762 runner.roles.fixer = Some(vec![fixer]);
20763 runner.roles.reviewers = vec![reviewer];
20764
20765 runner.sync_to_base().await.unwrap();
20766 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20767 assert_ne!(rebased, reviewed);
20768 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20769 if reentry {
20770 let shell = runner.state.config.shell();
20773 runner
20774 .stop_unresolved_review("deferred verification", &shell, &repo)
20775 .await
20776 .unwrap();
20777 assert!(runner.catchup_repairable());
20778 assert!(!calls.exists());
20779 runner.state = RunState::load(&runner.state.id).unwrap();
20780 assert!(runner.catchup_repairable());
20781 }
20782 runner.review_loop().await.unwrap();
20783 let first = &runner.state.reviews[0];
20784 assert_eq!(first.head, reviewed);
20785 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20786 assert!(
20787 first.verify_retried,
20788 "a reproduced build failure gets exactly one retry: {:?}",
20789 first.e2e
20790 );
20791 assert!(first.e2e[0].output_tail.contains("E0063"));
20792 assert!(!runner.catchup_repairable());
20793 if max_rounds == 2 {
20794 assert_eq!(
20795 runner.state.status,
20796 RunStatus::Gating,
20797 "{}",
20798 crate::report::run(&runner.state)
20799 );
20800 assert_eq!(runner.state.reviews.len(), 2);
20801 assert!(first.fix.as_ref().unwrap().committed);
20802 assert!(runner.state.reviews[1].clean);
20803 assert_eq!(
20804 runner
20805 .state
20806 .reviews
20807 .iter()
20808 .filter(|r| r.fix.is_some())
20809 .count(),
20810 1
20811 );
20812 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20813 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20814 } else {
20815 assert_eq!(runner.state.status, RunStatus::Blocked);
20816 assert!(first.fix.is_none());
20817 assert!(!calls.exists());
20818 let recovery = crate::recovery::assess(
20819 "prior",
20820 Some(&runner.state),
20821 &runner.landing_base(),
20822 crate::recovery::Relation::Diverged,
20823 );
20824 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20825 let reasons = recovery.reasons.join("\n");
20826 assert!(
20827 reasons.contains(&reviewed) && reasons.contains(&rebased),
20828 "{reasons}"
20829 );
20830 assert!(
20831 reasons.contains("base sync to")
20832 && reasons.contains("additional review is required"),
20833 "{reasons}"
20834 );
20835 }
20836 }
20837 }
20838
20839 #[tokio::test]
20851 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20852 crate::run::pin_test_home();
20853 let home = crate::run::home();
20854
20855 let tmp = tempfile::tempdir().expect("tempdir");
20856 let repo = tmp.path().join("repo");
20857 std::fs::create_dir_all(&repo).unwrap();
20858 init_repo(&repo);
20859 let head = crate::git::rev_parse(&repo, "HEAD")
20860 .await
20861 .expect("rev-parse");
20862 let cache_dir = tmp.path().join("target");
20863
20864 let mut config = Config::default();
20865 config.verify.e2e = vec![format!(
20866 "CARGO_TARGET_DIR='{}' test -f README.md",
20867 cache_dir.display()
20868 )];
20869 config.graph.review_rounds = 1;
20870 config.graph.timeout_verify = Some(2);
20871 config.graph.timeout_resource_wait = 2;
20872
20873 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20874 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20875 .expect("no io error acquiring directly")
20876 {
20877 crate::cache::AcquireOutcome::Acquired(g) => g,
20878 crate::cache::AcquireOutcome::Busy(b) => {
20879 panic!("expected the direct acquire to win the lease first: {b:?}")
20880 }
20881 };
20882
20883 let mut state = RunState::new(
20884 repo.clone(),
20885 "main".to_owned(),
20886 head.clone(),
20887 "task".to_owned(),
20888 config,
20889 );
20890 state.candidates = vec![Candidate {
20891 index: 0,
20892 label: 'A',
20893 agent: "alpha".to_owned(),
20894 branch: "does-not-exist".to_owned(),
20895 worktree: repo.clone(),
20896 summary: String::new(),
20897 stat: String::new(),
20898 files: 0,
20899 commits: 0,
20900 empty: false,
20901 failed: None,
20902 verified_noop: None,
20903 duration_ms: 0,
20904 folded: false,
20905 }];
20906 state.tally = Some(Tally {
20907 first_choice: BTreeMap::from([('A', 1)]),
20908 borda: BTreeMap::new(),
20909 winner: 'A',
20910 rankings: 1,
20911 unanimous_initial: true,
20912 deliberated: false,
20913 changed_votes: 0,
20914 unanimous_final: true,
20915 tie_break: None,
20916 judges: 0,
20917 present: 0,
20918 quorum: 0,
20919 met_quorum: true,
20920 uncontested: Some("only candidate A produced a change".to_owned()),
20921 });
20922 state.reviews = vec![ReviewRound {
20926 round: 1,
20927 head: head.clone(),
20928 verified_head: Some(head.clone()),
20929 verified_at: Some(jiff::Timestamp::now()),
20930 reviews: Vec::new(),
20931 e2e: vec![CommandOutcome {
20932 command: format!(
20933 "CARGO_TARGET_DIR='{}' test -f README.md",
20934 cache_dir.display()
20935 ),
20936 code: None,
20937 output_tail: "waiting for the shared build cache".to_owned(),
20938 duration_ms: 0,
20939 resource_blocked: true,
20940 }],
20941 fix: None,
20942 blocking: 1,
20943 answered: 1,
20944 expected: 1,
20945 clean: false,
20946 verify_retried: false,
20947 e2e_deferred: false,
20948 e2e_defer_reason: None,
20949 progressed: false,
20950 vote_split: false,
20951 reconsideration: Vec::new(),
20952 verdict: None,
20953 }];
20954
20955 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20956 let mut runner = Runner {
20957 state,
20958 roles: ResolvedRoles {
20959 implementers: Vec::new(),
20960 judges: Vec::new(),
20961 reviewers: Vec::new(),
20962 fixer: None,
20963 conductor: conductor(),
20964 implementer_roster: Vec::new(),
20965 judge_roster: Vec::new(),
20966 reviewer_roster: Vec::new(),
20967 },
20968 sem: Arc::new(Semaphore::new(1)),
20969 pause: Pause::new(),
20970 interrupt: Pause::new(),
20971 };
20972
20973 runner.review_loop().await.expect("review_loop");
20978
20979 assert_eq!(
20980 runner.state.reviews.len(),
20981 1,
20982 "no new round was started on top of the unresolved one: {:?}",
20983 runner.state.reviews
20984 );
20985 let last = &runner.state.reviews[0];
20986 assert_eq!(
20987 last.e2e_status(),
20988 E2eStatus::ResourceBlocked,
20989 "still contended: {last:?}"
20990 );
20991 assert!(
20992 last.verified_at.expect("still known") > first_attempt_at,
20993 "review_loop must have actually retried the check, not left it exactly as found"
20994 );
20995 assert_ne!(
20996 runner.state.status,
20997 RunStatus::Blocked,
20998 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20999 runner.state.status
21000 );
21001
21002 held.release();
21003 }
21004
21005 #[tokio::test]
21006 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21007 crate::run::pin_test_home();
21008 for status in [RunStatus::Blocked, RunStatus::Gating] {
21009 let mut runner = runner_at(status);
21010 runner.state.config.graph.land = true;
21011 runner.state.gate_ran = true;
21012 runner.state.merge = Some(MergeOutcome {
21013 mode: MergeMode::Pr,
21014 ok: false,
21015 detail: "the fixer produced no commit".into(),
21016 empty: false,
21017 });
21018 let url = "https://example.invalid/o/r/pull/1";
21019 runner.state.pr = Some(crate::run::PrRecord {
21020 url: url.into(),
21021 number: 1,
21022 state: "open".into(),
21023 checks: "red".into(),
21024 round: 1,
21025 rounds: 1,
21026 red_at_merge: Vec::new(),
21027 });
21028 runner.state.save().unwrap();
21029 runner.execute_graph().await.unwrap();
21033 assert_eq!(runner.state.status, RunStatus::Blocked);
21034 assert!(runner.state.land_sync_pending);
21035 assert!(runner.state.events.iter().any(|event|
21036 event.node == "land" && event.message == format!("watching {url}")));
21037 assert_eq!(
21038 runner.state.merge.as_ref().unwrap().detail,
21039 "the fixer produced no commit"
21040 );
21041 }
21042 }
21043
21044 #[tokio::test]
21045 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21046 crate::run::pin_test_home();
21047 let tmp = tempfile::tempdir().expect("tempdir");
21048 let repo = tmp.path().join("repo");
21049 std::fs::create_dir_all(&repo).unwrap();
21050 init_repo(&repo);
21051
21052 let mut config = Config::default();
21053 config.merge.mode = MergeMode::Pr;
21054 config.graph.land = true;
21055 config.graph.land_approval = false;
21056
21057 let mut state = RunState::new(
21058 repo.clone(),
21059 "main".to_owned(),
21060 "deadbeef".to_owned(),
21061 "task".to_owned(),
21062 config,
21063 );
21064 state.candidates = vec![Candidate {
21065 index: 0,
21066 label: 'A',
21067 agent: "alpha".to_owned(),
21068 branch: "does-not-exist".to_owned(),
21069 worktree: repo.clone(),
21070 summary: String::new(),
21071 stat: String::new(),
21072 files: 0,
21073 commits: 0,
21074 empty: false,
21075 failed: None,
21076 verified_noop: None,
21077 duration_ms: 0,
21078 folded: false,
21079 }];
21080 state.tally = Some(Tally {
21081 first_choice: BTreeMap::from([('A', 1)]),
21082 borda: BTreeMap::new(),
21083 winner: 'A',
21084 rankings: 1,
21085 unanimous_initial: true,
21086 deliberated: false,
21087 changed_votes: 0,
21088 unanimous_final: true,
21089 tie_break: None,
21090 judges: 0,
21091 present: 0,
21092 quorum: 0,
21093 met_quorum: true,
21094 uncontested: Some("only candidate A produced a change".to_owned()),
21095 });
21096 state.reviews = vec![ReviewRound {
21097 round: 1,
21098 head: "deadbeef".to_owned(),
21099 verified_head: None,
21100 verified_at: None,
21101 reviews: Vec::new(),
21102 e2e: Vec::new(),
21103 fix: None,
21104 blocking: 0,
21105 answered: 0,
21106 expected: 0,
21107 clean: true,
21108 verify_retried: false,
21109 e2e_deferred: false,
21110 e2e_defer_reason: None,
21111 progressed: false,
21112 vote_split: false,
21113 reconsideration: Vec::new(),
21114 verdict: None,
21115 }];
21116 state.gate = vec![CommandOutcome {
21117 command: "test".to_owned(),
21118 code: Some(0),
21119 output_tail: String::new(),
21120 duration_ms: 0,
21121 resource_blocked: false,
21122 }];
21123 state.gate_ran = true;
21124 state.status = RunStatus::Landing;
21128 state.merge = Some(MergeOutcome {
21129 mode: MergeMode::Pr,
21130 ok: true,
21131 detail: "https://example.invalid/x/y/pull/1".to_owned(),
21132 empty: false,
21133 });
21134
21135 ask_test_home();
21139 state.save().expect("save landing run");
21142 let store = ask::Questions::open();
21143 let q = ask_open_question(&store, &state.id);
21144
21145 let mut runner = Runner {
21146 state,
21147 roles: ResolvedRoles {
21148 implementers: Vec::new(),
21149 judges: Vec::new(),
21150 reviewers: Vec::new(),
21151 fixer: None,
21152 conductor: conductor(),
21153 implementer_roster: Vec::new(),
21154 judge_roster: Vec::new(),
21155 reviewer_roster: Vec::new(),
21156 },
21157 sem: Arc::new(Semaphore::new(1)),
21158 pause: Pause::new(),
21159 interrupt: Pause::new(),
21160 };
21161
21162 runner.execute().await.expect("execute");
21167
21168 assert_eq!(
21169 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21170 Some("https://example.invalid/x/y/pull/1"),
21171 "reentry must not push again or open a second pull request over the \
21172 one `land` is already watching"
21173 );
21174 assert_ne!(
21175 runner.state.status,
21176 RunStatus::Landing,
21177 "land could not actually reach the fake pull request, so it must \
21178 have given up rather than left the run silently parked forever"
21179 );
21180 assert_eq!(runner.state.status, RunStatus::Blocked);
21184 assert!(
21185 store.get(&q.id).unwrap().status.open(),
21186 "Blocked is still alive; settle_questions must have been a no-op here"
21187 );
21188 }
21189
21190 fn state_with_round(round: ReviewRound) -> RunState {
21191 let mut s = RunState::new(
21192 PathBuf::from("/repo"),
21193 "main".to_owned(),
21194 "abc1234".to_owned(),
21195 "add retries".to_owned(),
21196 Config::default(),
21197 );
21198 s.reviews = vec![round];
21199 s
21200 }
21201
21202 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21203 crate::verdict::Finding {
21204 id: id.to_owned(),
21205 severity,
21206 file: None,
21207 line: None,
21208 title: title.to_owned(),
21209 detail: String::new(),
21210 }
21211 }
21212
21213 #[test]
21214 fn pr_body_names_open_findings_and_declined_ones() {
21215 let round = ReviewRound {
21216 round: 2,
21217 head: "deadbee".to_owned(),
21218 verified_head: None,
21219 verified_at: None,
21220 reviews: vec![ReviewRecord {
21221 attempts: 0,
21222 reviewer: 1,
21223 agent: "alpha".to_owned(),
21224 summary: String::new(),
21225 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21226 vote: None,
21227 failed: None,
21228 duration_ms: 0,
21229 }],
21230 e2e: vec![CommandOutcome {
21231 command: "cargo test".to_owned(),
21232 code: Some(0),
21233 output_tail: String::new(),
21234 duration_ms: 0,
21235 resource_blocked: false,
21236 }],
21237 verify_retried: false,
21238 e2e_deferred: false,
21239 e2e_defer_reason: None,
21240 fix: Some(FixRecord {
21241 agent: "alpha".to_owned(),
21242 addressed: Vec::new(),
21243 rejected: vec![crate::verdict::Rejection {
21244 id: "R1-1-1".to_owned(),
21245 why: "not reachable from any caller".to_owned(),
21246 }],
21247 notes: String::new(),
21248 committed: true,
21249 failed: None,
21250 duration_ms: 0,
21251 continuation: None,
21252 }),
21253 blocking: 0,
21254 answered: 1,
21255 expected: 1,
21256 clean: false,
21257 progressed: true,
21258 vote_split: false,
21259 reconsideration: Vec::new(),
21260 verdict: None,
21261 };
21262 let state = state_with_round(round);
21263 let body = pr_message(&state, 'A').body;
21264
21265 assert!(body.contains("add retries"), "the task must still be there");
21266 assert!(body.contains("R2-1-1"), "{body}");
21267 assert!(!body.contains("unused import"), "{body}");
21268 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21269 assert!(
21270 !body.contains("not reachable from any caller"),
21271 "the reason it was declined: {body}"
21272 );
21273 }
21274
21275 #[test]
21276 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21277 let round = ReviewRound {
21278 round: 1,
21279 head: "deadbee".to_owned(),
21280 verified_head: None,
21281 verified_at: None,
21282 reviews: vec![ReviewRecord {
21283 attempts: 0,
21284 reviewer: 1,
21285 agent: "alpha".to_owned(),
21286 summary: String::new(),
21287 findings: Vec::new(),
21288 vote: None,
21289 failed: None,
21290 duration_ms: 0,
21291 }],
21292 e2e: Vec::new(),
21293 verify_retried: false,
21294 e2e_deferred: false,
21295 e2e_defer_reason: None,
21296 fix: None,
21297 blocking: 0,
21298 answered: 1,
21299 expected: 1,
21300 clean: true,
21301 progressed: false,
21302 vote_split: false,
21303 reconsideration: Vec::new(),
21304 verdict: None,
21305 };
21306 let state = state_with_round(round);
21307 let body = pr_message(&state, 'A').body;
21308 assert!(!body.contains("Open review findings"), "{body}");
21309 assert!(!body.contains("Declined"), "{body}");
21310 }
21311
21312 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21313 let mut state = RunState::new(
21314 PathBuf::from("/repo"),
21315 "main".to_owned(),
21316 "abc1234".to_owned(),
21317 instruction.to_owned(),
21318 Config::default(),
21319 );
21320 state.config.disk.min_free_bytes = 0;
21322 state.candidates.push(Candidate {
21323 index: 0,
21324 label: 'A',
21325 agent: "alpha".to_owned(),
21326 branch: "magi/x/A".to_owned(),
21327 worktree: PathBuf::from("/wt"),
21328 summary: summary.to_owned(),
21329 stat: String::new(),
21330 files: 1,
21331 commits: 1,
21332 empty: false,
21333 failed: None,
21334 verified_noop: None,
21335 folded: false,
21336 duration_ms: 0,
21337 });
21338 state
21339 }
21340
21341 fn review_state(subjects: &[&str]) -> RunState {
21342 let mut state = state_with_summary(
21343 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21344 "",
21345 );
21346 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21347 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21348 state
21349 }
21350
21351 #[tokio::test]
21354 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21355 ask_test_home();
21356 let tmp = tempfile::tempdir().unwrap();
21357 let repo = tmp.path().join("repo");
21358 std::fs::create_dir_all(&repo).unwrap();
21359 init_repo(&repo);
21360 let origin = tmp.path().join("origin.git");
21361 let g = |dir: &Path, args: &[&str]| {
21362 let out = std::process::Command::new("git")
21363 .args(args)
21364 .current_dir(dir)
21365 .quiet()
21366 .output()
21367 .expect("spawn git");
21368 assert!(
21369 out.status.success(),
21370 "git {args:?}: {}",
21371 String::from_utf8_lossy(&out.stderr)
21372 );
21373 };
21374 g(
21375 tmp.path(),
21376 &[
21377 "clone",
21378 "--bare",
21379 "-q",
21380 repo.to_str().unwrap(),
21381 origin.to_str().unwrap(),
21382 ],
21383 );
21384 g(
21385 &repo,
21386 &["remote", "add", "origin", origin.to_str().unwrap()],
21387 );
21388 let c1 = git::rev_parse(&repo, "main").await.unwrap();
21389
21390 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21392 g(&repo, &["add", "-A"]);
21393 g(
21394 &repo,
21395 &["commit", "-q", "-m", "chore(deps): update a crate"],
21396 );
21397 g(&repo, &["push", "-q", "origin", "main"]);
21398 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21399 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21400 g(&repo, &["add", "-A"]);
21401 g(
21402 &repo,
21403 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21404 );
21405 g(&repo, &["checkout", "-q", "main"]);
21406
21407 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21408 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21409 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21411 assert_eq!(fallback, c1);
21412
21413 let mut state = review_state(&[
21414 "chore(deps): update a crate",
21415 "fix(daemon): apply a chosen action",
21416 ]);
21417 state.repo = repo.clone();
21418 state.base_branch = "main".to_owned();
21419 state.base_commit = c1;
21420 refresh_reviewed_commits(&mut state, "feat/own").await;
21421 assert_eq!(
21422 state.reviewed_commits,
21423 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21424 );
21425 assert_eq!(
21426 review_title(&state).as_deref(),
21427 Some("fix(daemon): apply a chosen action")
21428 );
21429 assert_eq!(
21430 leaked_subjects(&state, "feat/own").await,
21431 Some(vec!["chore(deps): update a crate".to_owned()])
21432 );
21433 state.config.merge.remote = "nowhere".to_owned();
21436 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21437 }
21438
21439 struct SummaryFixture {
21443 _tmp: tempfile::TempDir,
21444 repo: PathBuf,
21445 reply: PathBuf,
21446 runner: Runner,
21447 }
21448
21449 fn summary_fixture() -> SummaryFixture {
21450 ask_test_home();
21451 let tmp = tempfile::tempdir().unwrap();
21452 let repo = tmp.path().join("repo");
21453 std::fs::create_dir_all(&repo).unwrap();
21454 init_repo(&repo);
21455 let origin = tmp.path().join("origin.git");
21456 let g = |args: &[&str]| {
21457 let out = std::process::Command::new("git")
21458 .args(args)
21459 .current_dir(&repo)
21460 .quiet()
21461 .output()
21462 .expect("spawn git");
21463 assert!(
21464 out.status.success(),
21465 "git {args:?}: {}",
21466 String::from_utf8_lossy(&out.stderr)
21467 );
21468 };
21469 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21470 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21471 g(&["push", "-q", "origin", "main"]);
21472 g(&["checkout", "-q", "-b", "feat/own"]);
21473 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21474 g(&["add", "-A"]);
21475 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21476 let reply = tmp.path().join("reply.txt");
21477 std::fs::write(
21478 &reply,
21479 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21480 )
21481 .unwrap();
21482 let mut runner = runner_at(RunStatus::Gating);
21483 let mut state = review_state(&["feat(own): add the own file"]);
21484 state.repo = repo.clone();
21485 state.base_branch = "main".to_owned();
21486 state.candidates[0].branch = "feat/own".to_owned();
21487 state.candidates[0].worktree = repo.clone();
21488 let mut author = spec("alpha");
21489 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21490 author
21491 .env
21492 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21493 state.config.agents = vec![author];
21494 runner.state = state;
21495 SummaryFixture {
21496 _tmp: tmp,
21497 repo,
21498 reply,
21499 runner,
21500 }
21501 }
21502
21503 async fn posted(f: &mut SummaryFixture) -> PrMessage {
21504 let winner = f.runner.state.candidates[0].clone();
21505 f.runner
21506 .guarded_pr_message(&winner, None, true)
21507 .await
21508 .unwrap()
21509 .expect("an English description never parks")
21510 }
21511
21512 fn clean_round_with_minor_findings() -> ReviewRound {
21513 let mut round = passing_round(2, None);
21514 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21515 round.reviews = vec![ReviewRecord {
21516 attempts: 0,
21517 reviewer: 1,
21518 agent: "alpha".into(),
21519 summary: "内部レビュー".into(),
21520 findings: vec![
21521 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21522 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21523 ],
21524 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21525 failed: None,
21526 duration_ms: 0,
21527 }];
21528 round
21529 }
21530
21531 #[tokio::test]
21532 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21533 for review_only in [false, true] {
21534 let mut f = summary_fixture();
21535 if !review_only {
21536 f.runner.state.candidates[0].agent = "alpha".into();
21537 }
21538 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21539 assert!(f.runner.state.reviews[0].clean);
21540 assert!(f.runner.state.reviews[0].fix.is_none());
21541 assert!(f.runner.state.open_findings().is_empty());
21542 assert_eq!(
21543 crate::pr_outcome::required_finding_ids(&f.runner.state),
21544 ["R2-1-1", "R2-1-2"]
21545 );
21546 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21547 let evidence = crate::pr_outcome::summary_evidence(
21548 &f.runner.state,
21549 crate::pr_outcome::FinalHead::At(&head),
21550 );
21551 assert!(
21552 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21553 );
21554 let omitted = posted(&mut f).await;
21556 assert!(omitted.body.contains("No description"));
21557 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21558 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21559 assert!(f.runner.state.change_summary.is_none());
21560 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21561 let described = posted(&mut f).await;
21562 assert!(described.body.contains("improve documentation for callers"));
21563 assert!(described.body.contains("remaining edge case"));
21564 assert!(described.body.contains("## Open review findings"));
21565 assert!(!described.body.contains("未修正"));
21566 }
21567 }
21568
21569 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21570 let mut f = summary_fixture();
21571 if !review_only {
21572 f.runner.state.candidates[0].agent = "alpha".into();
21573 }
21574 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21575 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21576 fixed
21577 .fix
21578 .as_mut()
21579 .unwrap()
21580 .rejected
21581 .push(crate::verdict::Rejection {
21582 id: "R1-1-1".into(),
21583 why: "呼び出されないコードのため辞退".into(),
21584 });
21585 let mut clean = clean_round_with_minor_findings();
21586 clean.verified_head = Some(head.clone());
21587 f.runner.state.reviews = vec![fixed, clean];
21588 ok_gate(&mut f.runner.state, Some(&head));
21589 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21590 f.runner.state.config.agents.push(scripted("beta", reply));
21591 f.runner.state.config.roles.language_judge =
21592 Some(crate::config::AgentChoice::One("beta".into()));
21593 f
21594 }
21595
21596 #[tokio::test]
21597 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21598 for review_only in [false, true] {
21599 let mut f = rewrite_fixture_with_findings(review_only,
21600 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21601 let m = posted(&mut f).await;
21602 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21603 let expected =
21604 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21605 assert!(m.body.contains(&expected));
21606 assert!(m.body.contains("## Review rounds"));
21607 assert!(
21608 m.body
21609 .contains("Gate (`verify.gate`): passed on the final commit")
21610 );
21611 assert!(
21612 m.body
21613 .contains("Tests (`verify.e2e`): passed on the final commit")
21614 );
21615 let c = f.runner.state.change_summary.as_ref().unwrap();
21616 assert!(
21617 m.body
21618 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21619 );
21620 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21621 assert!(crate::pr_outcome::covers_required_findings(
21622 &f.runner.state,
21623 &m.body
21624 ));
21625 assert!(m.body.contains("documentation remains incomplete"));
21626 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21627 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21628 assert!(f.runner.state.github_text.is_none());
21629 }
21630 }
21631
21632 #[tokio::test]
21633 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21634 for review_only in [false, true] {
21635 for reply in [
21636 "TITLE: fix: public description\nA short English description.",
21637 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21638 ] {
21639 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21640 let winner = f.runner.state.candidates[0].clone();
21641 let posted = f
21642 .runner
21643 .guarded_pr_message(&winner, None, true)
21644 .await
21645 .unwrap();
21646 assert!(posted.is_none());
21647 assert!(f.runner.state.parked);
21648 let withheld = f.runner.state.github_text.as_ref().unwrap();
21649 assert!(
21650 withheld.rewrite.is_none(),
21651 "an incomplete rewrite must not be selectable"
21652 );
21653 let q = ask::Questions::open()
21654 .get(withheld.question.as_ref().unwrap())
21655 .unwrap();
21656 assert!(
21657 !q.choices
21658 .contains(&crate::github_text::USE_REWRITE.to_owned())
21659 );
21660 assert!(
21661 q.detail.contains("R1-1-1")
21662 && q.detail.contains("R2-1-1")
21663 && q.detail.contains("R2-1-2")
21664 );
21665 }
21666 }
21667 }
21668
21669 #[tokio::test]
21670 async fn japanese_reports_stay_local_in_both_public_run_modes() {
21671 for review_only in [false, true] {
21672 let mut f = summary_fixture();
21673 if !review_only {
21674 f.runner.state.candidates[0].agent = "alpha".into();
21675 }
21676 f.runner.state.instruction = "内部タスクの説明".into();
21677 f.runner.state.candidates[0].summary = "内部の候補報告".into();
21678 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21679 f.runner.state.reviews = vec![
21680 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21681 round_with_notes(2, Some("二回目の内部修正報告です。")),
21682 passing_round(3, Some(&head)),
21683 ];
21684 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21685 ok_gate(&mut f.runner.state, Some(&head));
21686 let captured = f.repo.join("public-prompt.txt");
21687 let author = &mut f.runner.state.config.agents[0];
21688 author.command = vec![
21689 "sh".into(),
21690 "-c".into(),
21691 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21692 ];
21693 author.env.insert(
21694 "PROMPT_FILE".into(),
21695 captured.to_string_lossy().into_owned(),
21696 );
21697 let m = posted(&mut f).await;
21698 assert!(m.body.contains("Adds `own.txt`"));
21699 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21700 assert!(m.body.contains("R1-1-1"));
21701 assert!(
21702 m.body
21703 .contains("Gate (`verify.gate`): passed on the final commit")
21704 );
21705 assert!(
21706 m.body
21707 .contains("Tests (`verify.e2e`): passed on the final commit")
21708 );
21709 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21710 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21711 let prompt = std::fs::read_to_string(captured).unwrap();
21712 assert!(prompt.contains(&format!(
21713 "git diff {} {head}",
21714 f.runner.state.change_summary.as_ref().unwrap().base
21715 )));
21716 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21717 assert_eq!(
21718 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21719 "二回目の内部修正報告です。"
21720 );
21721 }
21722 }
21723
21724 #[tokio::test]
21725 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21726 for review_only in [false, true] {
21727 let mut f = summary_fixture();
21728 if !review_only {
21729 f.runner.state.candidates[0].agent = "alpha".into();
21730 }
21731 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21732 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21733 std::fs::write(&f.reply, "no usable description").unwrap();
21734 let failed = posted(&mut f).await;
21735 assert!(
21736 failed
21737 .body
21738 .contains("No description of what this change does could be generated")
21739 );
21740 assert!(failed.body.is_ascii());
21741 assert!(f.runner.state.github_text.is_none());
21742 assert!(f.runner.state.change_summary.is_none());
21743 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21744 let evidence = crate::pr_outcome::summary_evidence(
21745 &f.runner.state,
21746 crate::pr_outcome::FinalHead::At(&head),
21747 );
21748 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21749 assert!(
21750 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21751 .exists()
21752 );
21753 f.runner.state =
21754 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21755 std::fs::write(
21756 &f.reply,
21757 "TITLE: feat(own): add the own file\nAdds the committed own file.",
21758 )
21759 .unwrap();
21760 let retried = posted(&mut f).await;
21761 assert!(retried.body.contains("Adds the committed own file."));
21762 assert_eq!(
21763 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21764 "日本語の修正報告"
21765 );
21766 }
21767 }
21768
21769 #[tokio::test]
21770 async fn same_head_verification_and_findings_invalidate_public_prose() {
21771 let mut f = summary_fixture();
21772 posted(&mut f).await;
21773 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21774 let original = f
21775 .runner
21776 .state
21777 .change_summary
21778 .as_ref()
21779 .unwrap()
21780 .evidence
21781 .clone();
21782 ok_gate(&mut f.runner.state, Some(&head));
21783 std::fs::write(
21784 &f.reply,
21785 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21786 )
21787 .unwrap();
21788 let verified = posted(&mut f).await;
21789 assert!(verified.body.contains("now verified"));
21790 assert_ne!(
21791 f.runner.state.change_summary.as_ref().unwrap().evidence,
21792 original
21793 );
21794 let mut round = round_with_notes(1, Some("内部報告"));
21795 round.reviews.push(ReviewRecord {
21796 attempts: 0,
21797 reviewer: 1,
21798 agent: "alpha".into(),
21799 summary: "内部レビュー".into(),
21800 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21801 vote: None,
21802 failed: None,
21803 duration_ms: 0,
21804 });
21805 round
21806 .fix
21807 .as_mut()
21808 .unwrap()
21809 .rejected
21810 .push(crate::verdict::Rejection {
21811 id: "R1-1-1".into(),
21812 why: "呼び出し元から到達できない".into(),
21813 });
21814 f.runner.state.reviews = vec![round];
21815 let omitted = posted(&mut f).await;
21817 assert!(omitted.body.contains("No description"));
21818 assert!(omitted.body.contains("R1-1-1"));
21819 assert!(!omitted.body.contains("到達"));
21820 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21821 let described = posted(&mut f).await;
21822 assert!(described.body.contains("unreachable from callers"));
21823 assert!(described.body.contains("R1-1-1"));
21824 assert!(!described.body.contains("到達"));
21825 let evidence = f
21826 .runner
21827 .state
21828 .change_summary
21829 .as_ref()
21830 .unwrap()
21831 .evidence
21832 .clone();
21833 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21834 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21835 assert!(posted(&mut f).await.body.contains("rationale updated"));
21836 assert_ne!(
21837 f.runner.state.change_summary.as_ref().unwrap().evidence,
21838 evidence
21839 );
21840 f.runner
21842 .state
21843 .change_summary
21844 .as_mut()
21845 .unwrap()
21846 .evidence
21847 .clear();
21848 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21849 assert!(posted(&mut f).await.body.contains("current specification"));
21850 }
21851
21852 #[tokio::test]
21853 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21854 let mut f = summary_fixture();
21855 let m = posted(&mut f).await;
21856 assert_eq!(m.title, "feat(own): add the own file");
21857 assert!(
21858 m.body
21859 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21860 "{}",
21861 m.body
21862 );
21863 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21864 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21865 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21866 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21867 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21868 let kept = f.runner.state.change_summary.as_ref().unwrap();
21869 assert_eq!(
21870 (kept.head.as_str(), kept.base.as_str()),
21871 (head.as_str(), base.as_str())
21872 );
21873 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21874 }
21875
21876 #[tokio::test]
21877 async fn review_only_summary_receives_the_configured_github_language() {
21878 let mut f = summary_fixture();
21879 f.runner.state.config.graph.github_language = "ja".into();
21880 let captured = f.repo.join("summary-prompt.txt");
21881 let author = &mut f.runner.state.config.agents[0];
21882 author.command = vec![
21883 "sh".into(),
21884 "-c".into(),
21885 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21886 ];
21887 author.env.insert(
21888 "PROMPT_FILE".into(),
21889 captured.to_string_lossy().into_owned(),
21890 );
21891 std::fs::write(
21892 &f.reply,
21893 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21894 )
21895 .unwrap();
21896 let m = posted(&mut f).await;
21897 let prompt = std::fs::read_to_string(captured).unwrap();
21898 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21899 assert!(!prompt.contains("in English"));
21900 assert_eq!(m.title, "feat(own): ファイルを追加");
21901 assert!(m.body.contains("own.txt を追加します。"));
21902 }
21903
21904 #[tokio::test]
21905 async fn review_fixes_come_after_the_whole_change_summary() {
21906 let mut f = summary_fixture();
21907 f.runner.state.reviews = vec![
21908 round_with_notes(1, Some("handled the PATH case")),
21909 round_with_notes(2, None),
21910 ];
21911 let m = posted(&mut f).await;
21912 let summary = m.body.find("## Summary").expect("summary");
21913 let fixes = m.body.find("## Review rounds").expect("review rounds");
21914 assert!(summary < fixes, "{}", m.body);
21915 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21916 assert!(m.body.contains("## Final verification"), "{}", m.body);
21917 }
21918
21919 #[tokio::test]
21920 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21921 let mut f = summary_fixture();
21922 let first = posted(&mut f).await;
21923 let first_head = f.runner.state.change_summary.clone().unwrap().head;
21924 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21926 let again = posted(&mut f).await;
21927 assert_eq!((again.title, again.body), (first.title, first.body));
21928
21929 let g = |args: &[&str]| {
21930 let out = std::process::Command::new("git")
21931 .args(args)
21932 .current_dir(&f.repo)
21933 .quiet()
21934 .output()
21935 .expect("spawn git");
21936 assert!(out.status.success(), "git {args:?}");
21937 };
21938 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21939 g(&["add", "-A"]);
21940 g(&["commit", "-q", "-m", "fix(own): handle more"]);
21941 let second = posted(&mut f).await;
21942 let kept = f.runner.state.change_summary.clone().unwrap();
21943 assert_ne!(kept.head, first_head);
21944 assert!(second.body.contains("Not read yet."), "{}", second.body);
21945 assert!(!second.body.contains("so the first summary is read"));
21946 assert_eq!(second.title, "feat(own): other");
21947 }
21948
21949 #[tokio::test]
21950 async fn an_unavailable_summary_is_stated_and_never_invented() {
21951 let mut f = summary_fixture();
21952 std::fs::write(&f.reply, "no title line here").unwrap();
21953 let m = posted(&mut f).await;
21954 assert!(
21955 m.body
21956 .contains("No description of what this change does could be generated"),
21957 "{}",
21958 m.body
21959 );
21960 assert!(m.body.contains("internal reports remain in local records"));
21961 assert!(f.runner.state.change_summary.is_none());
21962 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21963 let winner = f.runner.state.candidates[0].clone();
21965 let preview = f
21966 .runner
21967 .guarded_pr_message(&winner, None, false)
21968 .await
21969 .unwrap()
21970 .unwrap();
21971 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21972 }
21973
21974 #[test]
21975 fn pr_message_review_single_commit_uses_its_subject() {
21976 let state = review_state(&["feat(nats): per-role user"]);
21977 let m = pr_message(&state, 'A');
21978 assert_eq!(m.title, "feat(nats): per-role user");
21979 assert!(!m.body.contains("Review the work already"), "{}", m.body);
21980 assert!(m.body.contains("## Commits under review"), "{}", m.body);
21981 }
21982
21983 #[test]
21984 fn pr_message_review_multi_commit_takes_the_oldest() {
21985 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21986 let m = pr_message(&state, 'A');
21987 assert_eq!(m.title, "feat: the change");
21988 for s in ["feat: the change", "fix: typo", "fix: again"] {
21989 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21990 }
21991 }
21992
21993 #[test]
21994 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21995 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21996 let mut state = review_state(&[first, "fix: later fixup"]);
21997 state.candidates[0].branch = "機能/ログイン".to_owned();
21998 let m = pr_message(&state, 'A');
21999 assert!(
22000 m.title.starts_with("chore: land candidate A of run"),
22001 "{}",
22002 m.title
22003 );
22004 }
22005 }
22006
22007 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22008 BranchFacts {
22009 commits: commits
22010 .iter()
22011 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22012 .collect(),
22013 stat: stat.to_owned(),
22014 }
22015 }
22016
22017 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22018 let mut r = review_round(true, 0, 1, 1, true, true);
22019 r.round = round;
22020 r.fix = notes.map(|n| FixRecord {
22021 agent: "fixer".to_owned(),
22022 addressed: Vec::new(),
22023 rejected: Vec::new(),
22024 notes: n.to_owned(),
22025 committed: true,
22026 failed: None,
22027 duration_ms: 0,
22028 continuation: None,
22029 });
22030 r
22031 }
22032
22033 #[test]
22034 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22035 let state = review_state(&["ignored"]);
22036 let f = facts(
22037 &[
22038 (
22039 "fix(login): resolve PATH on macOS",
22040 "Login shells skip rc files.",
22041 ),
22042 ("fix: address review", ""),
22043 ],
22044 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22045 );
22046 let m = pr_message_with(&state, 'A', Some(&f));
22047 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22048 assert!(
22049 m.body
22050 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
22051 "{}",
22052 m.body
22053 );
22054 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22055 assert!(m.body.contains("## Diff stat"), "{}", m.body);
22056 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22057 for banned in [
22058 "Review the work already",
22059 "no task statement",
22060 "Original task",
22061 ] {
22062 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22063 }
22064 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22065 }
22066
22067 #[test]
22068 fn pr_message_review_truncates_a_huge_first_commit_body() {
22069 let state = review_state(&["ignored"]);
22070 let f = facts(
22071 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22072 "s",
22073 );
22074 let m = pr_message_with(&state, 'A', Some(&f));
22075 assert!(m.body.len() < 30_000, "{}", m.body.len());
22076 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22077 assert!(
22078 m.body.contains("- ... 1 more commit(s)"),
22079 "{}",
22080 m.body.len()
22081 );
22082 assert!(m.body.ends_with("magi:candidate-a\n"));
22083 }
22084
22085 #[test]
22086 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22087 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22088 assert_eq!(m.title, "feat: x");
22089 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22090 assert!(!m.body.contains("Diff stat"), "{}", m.body);
22091 assert!(!m.body.contains("no task statement"), "{}", m.body);
22092 }
22093
22094 #[test]
22095 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22096 let mut state = review_state(&["日本語の件名"]);
22097 state.candidates[0].branch = "fix/macos-login-path".to_owned();
22098 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22099 state.candidates[0].branch = "機能/ログイン".to_owned();
22100 assert!(
22101 pr_message(&state, 'A')
22102 .title
22103 .starts_with("chore: land candidate A")
22104 );
22105 }
22106
22107 #[test]
22108 fn pr_message_review_fixes_survive_a_clean_final_round() {
22109 let mut state = review_state(&["feat: x"]);
22110 state.reviews = vec![
22111 round_with_notes(1, Some("handled the PATH case")),
22112 round_with_notes(2, None),
22113 ];
22114 let body = pr_message(&state, 'A').body;
22115 assert!(
22116 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22117 "{body}"
22118 );
22119 assert!(body.contains("### Round 1"), "{body}");
22120
22121 state.reviews = vec![
22122 round_with_notes(1, Some("first fix")),
22123 round_with_notes(2, Some("")),
22124 round_with_notes(3, Some("second fix")),
22125 round_with_notes(4, None),
22126 ];
22127 let body = pr_message(&state, 'A').body;
22128 assert!(
22129 body.contains("### Round 1") && !body.contains("first fix"),
22130 "{body}"
22131 );
22132 assert!(
22133 body.contains("### Round 3") && !body.contains("second fix"),
22134 "{body}"
22135 );
22136 assert!(body.contains("### Round 2"), "{body}");
22137 }
22138
22139 const FINAL: &str = "abcdef1234567890";
22140
22141 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22142 let mut r = round_with_notes(round, None);
22143 r.verified_head = verified.map(str::to_owned);
22144 r
22145 }
22146
22147 fn ok_gate(state: &mut RunState, head: Option<&str>) {
22148 state.gate = vec![CommandOutcome {
22149 command: "gate".to_owned(),
22150 code: Some(0),
22151 output_tail: String::new(),
22152 duration_ms: 0,
22153 resource_blocked: false,
22154 }];
22155 state.gate_ran = true;
22156 state.gate_head = head.map(str::to_owned);
22157 }
22158
22159 fn body_at(state: &RunState) -> String {
22160 pr_message_raw(
22161 state,
22162 'A',
22163 None,
22164 crate::pr_outcome::FinalHead::At(FINAL),
22165 ChangeNote::NotAsked,
22166 )
22167 .body
22168 }
22169
22170 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22171
22172 #[test]
22173 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22174 let mut state = review_state(&["feat: x"]);
22175 state.reviews = vec![
22176 round_with_notes(1, Some(SANDBOXED)),
22177 passing_round(2, Some(FINAL)),
22178 ];
22179 ok_gate(&mut state, Some(FINAL));
22180 let body = body_at(&state);
22181 assert!(!body.contains("Handled the PATH case."), "{body}");
22182 assert!(!body.contains("could not commit"), "{body}");
22183 assert!(!body.contains("could not run"), "{body}");
22184 assert!(
22185 body.contains("The fix is committed on the branch"),
22186 "{body}"
22187 );
22188 assert!(!body.contains("sandbox"), "{body}");
22189 assert!(
22190 body.contains("Tests (`verify.e2e`): passed on the final commit"),
22191 "{body}"
22192 );
22193 assert!(
22194 body.contains("Gate (`verify.gate`): passed on the final commit"),
22195 "{body}"
22196 );
22197 assert!(
22199 state.reviews[0]
22200 .fix
22201 .as_ref()
22202 .unwrap()
22203 .notes
22204 .contains("could not commit")
22205 );
22206 }
22207
22208 #[test]
22209 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22210 let mut state = review_state(&["feat: x"]);
22211 let mut r1 = round_with_notes(1, Some(SANDBOXED));
22212 r1.fix.as_mut().unwrap().committed = false;
22213 let mut r2 = passing_round(2, Some(FINAL));
22214 r2.e2e[0].code = Some(1);
22215 state.reviews = vec![r1, r2];
22216 let body = body_at(&state);
22217 assert!(!body.contains("could not commit"), "{body}");
22218 assert!(!body.contains("could not run"), "{body}");
22219 assert!(body.contains("This round produced no commit"), "{body}");
22220 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22221 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22222 assert!(!body.contains("passed"), "{body}");
22223 }
22224
22225 #[test]
22226 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22227 let mut state = review_state(&["feat: x"]);
22228 state.reviews = vec![
22229 round_with_notes(1, Some(SANDBOXED)),
22230 passing_round(2, Some("0123456789abcdef")),
22231 ];
22232 ok_gate(&mut state, Some("0123456789abcdef"));
22233 let body = body_at(&state);
22234 assert!(!body.contains("HTTP tests could not run"), "{body}");
22235 assert!(!body.contains("could not commit"), "{body}");
22237 assert!(
22238 body.contains("passed on an earlier commit (`0123456`)"),
22239 "{body}"
22240 );
22241 assert!(!body.contains("passed on the final commit"), "{body}");
22242
22243 ok_gate(&mut state, None);
22245 let body = body_at(&state);
22246 assert!(
22247 body.contains("passed on a commit that was not recorded"),
22248 "{body}"
22249 );
22250 }
22251
22252 #[test]
22253 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22254 let state = state_with_summary(
22255 "add retries to the client",
22256 "TITLE: feat: retries\n\nDid it.",
22257 );
22258 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22259 assert_eq!(m.title, "feat: retries");
22260 assert!(
22261 m.body.contains("<summary>Original task</summary>"),
22262 "{}",
22263 m.body
22264 );
22265 assert!(!m.body.contains("Commits under review"), "{}", m.body);
22266 assert!(
22267 m.body
22268 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22269 "{}",
22270 m.body
22271 );
22272 }
22273
22274 #[test]
22275 fn pr_message_review_bounds_a_long_english_subject() {
22276 let long = format!("feat: {}", "word ".repeat(100));
22277 let m = pr_message(&review_state(&[&long]), 'A');
22278 assert!(m.title.starts_with("feat: word"), "{}", m.title);
22279 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22280 }
22281
22282 #[test]
22283 fn pr_message_implementation_run_is_unchanged_by_review_support() {
22284 let state = state_with_summary("add retries\n\ndetails", "- did some things");
22285 let m = pr_message(&state, 'A');
22286 assert_eq!(m.title, "add retries");
22287 assert!(m.body.contains("<summary>Original task</summary>"));
22288 assert!(!m.body.contains("Commits under review"));
22289 assert_eq!(landing_subject_source(&state), state.instruction);
22290 }
22291
22292 #[test]
22293 fn review_run_squash_subject_is_the_change_not_the_prompt() {
22294 let state = review_state(&["feat: the change", "fix: typo"]);
22295 let source = landing_subject_source(&state);
22296 assert_eq!(land::merge_subject("", &source), "feat: the change");
22297 assert_eq!(
22298 land::merge_subject("magi: candidate A (uncommitted work)", &source),
22299 "feat: the change"
22300 );
22301 assert_eq!(
22303 land::merge_subject("feat: renamed by hand", &source),
22304 "feat: renamed by hand"
22305 );
22306 let blank = review_state(&["日本語"]);
22307 assert!(
22308 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22309 );
22310 }
22311
22312 #[test]
22313 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22314 let state = review_state(&["feat: the change"]);
22315 let source = landing_subject_source(&state);
22316 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22317 assert_eq!(
22318 land::merge_subject(landing_title(&state, old), &source),
22319 "feat: the change"
22320 );
22321 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22322 let task = state_with_summary("add retries", "");
22323 assert_eq!(landing_title(&task, old), old);
22324 }
22325
22326 #[test]
22327 fn pr_message_describes_the_change_not_the_task() {
22328 let state = state_with_summary(
22329 "今回やってほしいこと: results projector を直す",
22330 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22331 );
22332 let m = pr_message(&state, 'A');
22333 assert_eq!(m.title, "fix(web): batch the runs list reads");
22334 assert!(
22335 m.body.starts_with("## Summary\n\n- reads run.json once"),
22336 "{}",
22337 m.body
22338 );
22339 assert!(!m.body.contains("TITLE:"), "{}", m.body);
22340 let task_at = m.body.find("今回やってほしいこと").unwrap();
22341 let details_at = m.body.find("<details>").unwrap();
22342 assert!(
22343 details_at < task_at,
22344 "the task lives inside <details>: {}",
22345 m.body
22346 );
22347 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22348 assert!(m.body.contains("magi:candidate-a"));
22349 }
22350
22351 #[test]
22352 fn pr_message_falls_back_to_the_task_without_a_title_line() {
22353 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22354 let m = pr_message(&state, 'A');
22355 assert_eq!(m.title, "add retries");
22356 assert!(
22357 m.body.contains("## Summary\n\n- did some things"),
22358 "{}",
22359 m.body
22360 );
22361
22362 let none = RunState::new(
22363 PathBuf::from("/repo"),
22364 "main".to_owned(),
22365 "abc1234".to_owned(),
22366 "add retries".to_owned(),
22367 Config::default(),
22368 );
22369 let m = pr_message(&none, 'A');
22370 assert_eq!(m.title, "add retries");
22371 assert!(!m.body.contains("## Summary"), "{}", m.body);
22372 }
22373
22374 #[test]
22375 fn pr_message_refuses_the_candidate_commit_subject() {
22376 for bad in [
22377 "TITLE: magi: candidate A (uncommitted work)",
22378 "TITLE: chore: stuff (uncommitted work)",
22379 "TITLE: ",
22380 ] {
22381 let state = state_with_summary("add retries", bad);
22382 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22383 }
22384 }
22385
22386 #[test]
22387 fn pr_message_bounds_a_very_long_task_and_title() {
22388 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22389 let state = state_with_summary(&long, "- nothing");
22390 let m = pr_message(&state, 'A');
22391 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22392 assert!(!m.title.contains('\n'));
22393
22394 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22395 let m = pr_message(&state, 'A');
22396 assert!(m.title.starts_with("feat: "));
22397 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22398 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22399 }
22400
22401 fn long_title_of(instruction: &str) -> String {
22402 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22403 }
22404
22405 #[test]
22406 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22407 let first = "Make the landing path keep a readable title for long tasks";
22408 let line = format!(
22409 "{first}. {}",
22410 "And then keep going with more words ".repeat(20)
22411 );
22412 let t = long_title_of(&line);
22413 assert_eq!(t, first);
22414 assert!(!t.starts_with("chore: land"));
22415 }
22416
22417 #[test]
22418 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22419 let line = "word ".repeat(200);
22420 let t = long_title_of(&line);
22421 assert!(t.ends_with("word..."), "{t}");
22422 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22423 }
22424
22425 #[test]
22426 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22427 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22428 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22429 }
22430 }
22431
22432 #[test]
22433 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22434 let mut state = state_with_summary("再試行を追加する", "- nothing");
22435 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22436 state.config.graph.github_language = "ja".into();
22437 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22438 }
22439
22440 #[test]
22441 fn pr_message_title_limit_is_exact() {
22442 let at = "a".repeat(PR_TITLE_MAX);
22443 assert_eq!(long_title_of(&at), at);
22444 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22445 assert!(over.ends_with("..."), "{over}");
22446 assert_eq!(over.chars().count(), PR_TITLE_MAX);
22447 }
22448
22449 #[test]
22450 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22451 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22452 let t = long_title_of(&line);
22453 assert!(t.ends_with("..."), "{t}");
22454 assert!(t.is_ascii(), "{t}");
22455 }
22456
22457 #[test]
22458 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22459 let line = format!(
22460 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22461 "plus more filler words ".repeat(20)
22462 );
22463 let t = long_title_of(&line);
22464 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22465 }
22466
22467 #[test]
22468 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22469 let line = format!("\n\n# {}", "title words ".repeat(40));
22470 let state = state_with_summary(&line, "- nothing");
22471 let m = pr_message(&state, 'A');
22472 assert!(m.title.starts_with("title words"), "{}", m.title);
22473 assert_eq!(
22474 land::merge_subject(&m.title, &landing_subject_source(&state)),
22475 m.title
22476 );
22477 assert_eq!(
22479 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22480 "feat: renamed by hand"
22481 );
22482 }
22483
22484 #[tokio::test]
22485 async fn github_text_rewrite_is_bounded_and_falls_back() {
22486 crate::run::pin_test_home();
22487 for (reply, accepted) in [
22488 (
22489 "TITLE: fix: retries\nAdd retries for failed requests.",
22490 true,
22491 ),
22492 (
22493 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22494 false,
22495 ),
22496 (
22497 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22498 true,
22499 ),
22500 (
22501 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22502 true,
22503 ),
22504 (
22505 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22506 true,
22507 ),
22508 (
22509 "TITLE: fix: retries\n日本語の説明です。 token=secret",
22510 false,
22511 ),
22512 ("TITLE: fix: retries\n", false),
22513 ] {
22514 let dir = tempfile::tempdir().unwrap();
22515 let mut runner = runner_at(RunStatus::Gating);
22516 runner.state = state_with_summary(
22517 "add retries",
22518 "TITLE: fix: retries\n日本語の説明を書きます。",
22519 );
22520 runner.state.repo = dir.path().to_owned();
22521 runner.state.candidates[0].worktree = dir.path().to_owned();
22522 let mut author = spec("alpha");
22523 author.command = vec![
22524 "sh".into(),
22525 "-c".into(),
22526 "printf '%s' \"$REWRITE_REPLY\"".into(),
22527 ];
22528 author.env.insert("REWRITE_REPLY".into(), reply.into());
22529 runner.state.config.agents = vec![author];
22530 seed_public_summary(&mut runner);
22531 let winner = runner.state.candidates[0].clone();
22532 let first = runner
22533 .guarded_pr_message(&winner, None, true)
22534 .await
22535 .unwrap();
22536 let message = if accepted {
22537 first.expect("a passing rewrite never asks")
22538 } else {
22539 assert!(first.is_none());
22541 assert!(runner.state.parked);
22542 let store = ask::Questions::open();
22543 let asked: Vec<_> = store
22544 .list()
22545 .into_iter()
22546 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22547 .collect();
22548 assert_eq!(asked.len(), 1);
22549 assert!(!asked[0].detail.contains("secret"));
22550 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22551 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22552 assert_eq!(
22553 asked[0]
22554 .choices
22555 .contains(&crate::github_text::USE_REWRITE.to_owned()),
22556 rewrite.is_some()
22557 );
22558 store
22561 .update(&asked[0].id, |q| {
22562 q.abandon("test");
22563 Ok(())
22564 })
22565 .unwrap();
22566 let again = runner
22567 .guarded_pr_message(&winner, None, true)
22568 .await
22569 .unwrap()
22570 .expect("an abandoned question falls back");
22571 let n = store
22572 .list()
22573 .into_iter()
22574 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22575 .count();
22576 assert_eq!(n, 1, "asked once per rejected text");
22577 again
22578 };
22579 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22580 assert!(!message.body.contains("secret"));
22581 assert!(!message.body.contains("prefix"));
22582 assert!(!message.title.contains("secret"));
22583 assert_eq!(
22584 message.body.contains("Add retries for failed requests."),
22585 accepted
22586 );
22587 assert_eq!(
22588 runner.state.seats["github-text-rewrite"].turns, 1,
22589 "only one rewrite invocation"
22590 );
22591 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22592 if accepted {
22593 assert!(runner.state.github_text.is_none());
22594 assert!(!runner.state.parked);
22595 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22596 assert!(message.body.contains("## Final verification"));
22597 assert!(message.body.contains("Evidence: final diff"));
22598 assert!(
22599 message
22600 .body
22601 .contains(&format!("magi:run/{}", runner.state.id))
22602 );
22603 }
22604 if !accepted {
22605 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22606 assert!(
22607 message
22608 .body
22609 .contains(&format!("magi:run/{}", runner.state.id))
22610 );
22611 }
22612 }
22613 }
22614
22615 fn scripted(id: &str, reply: &str) -> AgentSpec {
22616 let mut a = spec(id);
22617 a.command = vec![
22618 "sh".into(),
22619 "-c".into(),
22620 "printf '%s' \"$REWRITE_REPLY\"".into(),
22621 ];
22622 a.env.insert("REWRITE_REPLY".into(), reply.into());
22623 a
22624 }
22625
22626 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22627 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22628
22629 fn seed_public_summary(runner: &mut Runner) {
22631 let repo = &runner.state.repo;
22632 let g = |args: &[&str]| {
22633 let out = std::process::Command::new("git")
22634 .args(args)
22635 .current_dir(repo)
22636 .quiet()
22637 .output()
22638 .unwrap();
22639 assert!(out.status.success(), "git {args:?}");
22640 String::from_utf8_lossy(&out.stdout).trim().to_owned()
22641 };
22642 g(&["init", "-q", "-b", "main"]);
22643 g(&[
22644 "-c",
22645 "user.name=Test",
22646 "-c",
22647 "user.email=test@example.invalid",
22648 "commit",
22649 "-q",
22650 "--allow-empty",
22651 "-m",
22652 "feat: fixture",
22653 ]);
22654 let head = g(&["rev-parse", "HEAD"]);
22655 g(&["update-ref", "refs/remotes/origin/main", &head]);
22656 let evidence = crate::pr_outcome::summary_evidence(
22657 &runner.state,
22658 crate::pr_outcome::FinalHead::At(&head),
22659 );
22660 let generated = &runner.state.candidates[0].summary;
22661 runner.state.change_summary = Some(crate::run::ChangeSummary {
22662 base: head.clone(),
22663 head,
22664 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22665 title: summary_title(generated).unwrap(),
22666 body: summary_without_title(generated),
22667 });
22668 }
22669
22670 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22671 let mut runner = runner_at(RunStatus::Gating);
22672 runner.state = state_with_summary("add retries", JA_SUMMARY);
22673 if review_only {
22674 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22675 }
22676 runner.state.repo = dir.to_owned();
22677 runner.state.candidates[0].worktree = dir.to_owned();
22678 runner.state.config.agents = agents;
22679 seed_public_summary(&mut runner);
22680 runner
22681 }
22682
22683 fn gate_questions(runner: &Runner) -> usize {
22684 ask::Questions::open()
22685 .list()
22686 .into_iter()
22687 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22688 .count()
22689 }
22690
22691 #[tokio::test]
22692 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22693 crate::run::pin_test_home();
22694 let dir = tempfile::tempdir().unwrap();
22695 let mut runner = rewrite_runner(
22696 dir.path(),
22697 true,
22698 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22699 );
22700 runner.state.config.roles.language_judge =
22701 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22702 let winner = runner.state.candidates[0].clone();
22703 let msg = runner
22704 .guarded_pr_message(&winner, None, true)
22705 .await
22706 .unwrap()
22707 .expect("an English rewrite posts without asking");
22708 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22709 assert!(msg.body.contains("<details>kept</details>"));
22710 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22711 assert_eq!(gate_questions(&runner), 0);
22712 assert!(runner.state.github_text.is_none());
22713 assert!(!runner.state.seats.contains_key("impl-A"));
22714 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22715 }
22716
22717 #[tokio::test]
22718 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22719 crate::run::pin_test_home();
22720 let dir = tempfile::tempdir().unwrap();
22721 let mut runner = rewrite_runner(
22722 dir.path(),
22723 false,
22724 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22725 );
22726 runner.state.config.roles.language_judge =
22727 Some(crate::config::AgentChoice::One("beta".into()));
22728 runner.state.seats.insert(
22729 "impl-A".into(),
22730 SeatState::new("impl-A", "alpha", runner.state.seed),
22731 );
22732 let winner = runner.state.candidates[0].clone();
22733 let msg = runner
22734 .guarded_pr_message(&winner, None, true)
22735 .await
22736 .unwrap()
22737 .expect("rewritten");
22738 assert!(msg.body.contains("<details>kept</details>"));
22739 assert_eq!(runner.state.seats["impl-A"].turns, 0);
22740 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22741 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22742 }
22743
22744 #[tokio::test]
22745 async fn rewrite_reuses_a_resumable_implementer_session() {
22746 crate::run::pin_test_home();
22747 let dir = tempfile::tempdir().unwrap();
22748 let mut runner = rewrite_runner(
22749 dir.path(),
22750 false,
22751 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22752 );
22753 runner.state.config.roles.language_judge =
22754 Some(crate::config::AgentChoice::One("beta".into()));
22755 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22756 seat.turns = 1;
22757 runner.state.seats.insert("impl-A".into(), seat);
22758 let winner = runner.state.candidates[0].clone();
22759 runner
22760 .guarded_pr_message(&winner, None, true)
22761 .await
22762 .unwrap()
22763 .expect("rewritten");
22764 assert_eq!(runner.state.seats["impl-A"].turns, 2);
22765 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22766 }
22767
22768 #[tokio::test]
22769 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22770 crate::run::pin_test_home();
22773 let dir = tempfile::tempdir().unwrap();
22774 let mut exhausted = spec("alpha");
22775 exhausted.command = vec!["false".into()];
22776 let mut runner = rewrite_runner(
22777 dir.path(),
22778 false,
22779 vec![exhausted, scripted("beta", EN_REWRITE)],
22780 );
22781 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22783 "alpha".into(),
22784 "beta".into(),
22785 ]));
22786 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22787 seat.turns = 1;
22788 runner.state.seats.insert("impl-A".into(), seat);
22789 let winner = runner.state.candidates[0].clone();
22790 let msg = runner
22791 .guarded_pr_message(&winner, None, true)
22792 .await
22793 .unwrap()
22794 .expect("the role rewrites after the implementer fails");
22795 assert!(msg.body.contains("`src/net.rs`"));
22796 assert_eq!(
22797 runner.state.seats["impl-A"].turns, 2,
22798 "the implementer was tried once"
22799 );
22800 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22801 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22802 }
22803
22804 #[tokio::test]
22805 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22806 crate::run::pin_test_home();
22807 let dir = tempfile::tempdir().unwrap();
22808 let mut broken = spec("bad");
22809 broken.command = vec!["false".into()];
22810 let mut runner = rewrite_runner(
22811 dir.path(),
22812 true,
22813 vec![
22814 broken,
22815 scripted("noformat", "Add retries."),
22816 scripted("good", EN_REWRITE),
22817 ],
22818 );
22819 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22820 "bad".into(),
22821 "noformat".into(),
22822 "good".into(),
22823 ]));
22824 let winner = runner.state.candidates[0].clone();
22825 let msg = runner
22826 .guarded_pr_message(&winner, None, true)
22827 .await
22828 .unwrap()
22829 .expect("the third agent rewrites");
22830 assert!(msg.body.contains("`src/net.rs`"));
22831 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22832 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22833 }
22834
22835 #[tokio::test]
22836 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22837 crate::run::pin_test_home();
22838 for (agents, role, saved) in [
22840 (
22841 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22842 Some("beta"),
22843 true,
22844 ),
22845 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22846 (Vec::new(), None, false),
22847 ] {
22848 let dir = tempfile::tempdir().unwrap();
22849 let mut runner = rewrite_runner(dir.path(), true, agents);
22850 runner.state.config.roles.language_judge =
22851 role.map(|r| crate::config::AgentChoice::One(r.into()));
22852 let winner = runner.state.candidates[0].clone();
22853 let first = runner
22854 .guarded_pr_message(&winner, None, true)
22855 .await
22856 .unwrap();
22857 assert!(first.is_none(), "withheld text parks the run");
22858 assert!(runner.state.parked);
22859 assert_eq!(gate_questions(&runner), 1);
22860 let g = runner.state.github_text.as_ref().unwrap();
22861 assert_eq!(g.rewrite.is_some(), saved);
22862 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22863 }
22864 }
22865
22866 #[tokio::test]
22867 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22868 crate::run::pin_test_home();
22869 let dir = tempfile::tempdir().unwrap();
22870 let mut runner = runner_at(RunStatus::Gating);
22871 runner.state =
22872 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22873 runner.state.repo = dir.path().to_owned();
22874 runner.state.candidates[0].worktree = dir.path().to_owned();
22875 let mut author = spec("alpha");
22876 author.command = vec![
22877 "sh".into(),
22878 "-c".into(),
22879 "printf '%s' \"$REWRITE_REPLY\"".into(),
22880 ];
22881 author.env.insert(
22882 "REWRITE_REPLY".into(),
22883 format!(
22884 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22885 "English sentence. ".repeat(600)
22886 ),
22887 );
22888 runner.state.config.agents = vec![author];
22889 seed_public_summary(&mut runner);
22890 let winner = runner.state.candidates[0].clone();
22891 assert!(
22892 runner
22893 .guarded_pr_message(&winner, None, true)
22894 .await
22895 .unwrap()
22896 .is_none()
22897 );
22898 let record = runner.state.github_text.clone().unwrap();
22899 let rewrite = record.rewrite.clone().unwrap();
22900 assert_eq!(rewrite.failed, ["body-language"]);
22901 assert!(
22902 rewrite
22903 .body
22904 .ends_with(&format!("magi:run/{}", runner.state.id))
22905 );
22906 let store = ask::Questions::open();
22907 let q = store.get(record.question.as_ref().unwrap()).unwrap();
22908 assert!(
22909 q.choices
22910 .contains(&crate::github_text::USE_REWRITE.to_owned())
22911 );
22912 for label in [
22913 "ORIGINAL",
22914 "FALLBACK",
22915 "REWRITE",
22916 "Rules still failed: body-language",
22917 "truncated",
22918 ] {
22919 assert!(q.detail.contains(label), "{label}");
22920 }
22921 let artifact = std::fs::read_to_string(crate::run::artifact_path(
22922 &runner.state,
22923 &crate::github_text::artifact_name(&record.fingerprint),
22924 ))
22925 .unwrap();
22926 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22927 runner.state =
22929 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22930 store
22931 .update(&q.id, |q| {
22932 q.say("まだ日本語")?;
22933 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22934 })
22935 .unwrap();
22936 assert!(
22937 runner
22938 .guarded_pr_message(&winner, None, true)
22939 .await
22940 .unwrap()
22941 .is_none()
22942 );
22943 let retry = runner.state.github_text.as_ref().unwrap();
22944 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22945 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22946 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22947 assert!(
22948 q.posting_comparison
22949 .as_ref()
22950 .unwrap()
22951 .matches_detail(&q.detail)
22952 );
22953 store
22954 .update(&q.id, |q| {
22955 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22956 })
22957 .unwrap();
22958 let message = runner
22959 .guarded_pr_message(&winner, None, true)
22960 .await
22961 .unwrap()
22962 .unwrap();
22963 assert_eq!(message.title, rewrite.title);
22964 assert_eq!(message.body, rewrite.body);
22965 assert_eq!(
22966 runner.state.github_text.as_ref().unwrap().choice,
22967 Some(GithubTextChoice::Rewrite)
22968 );
22969 let again = runner
22970 .guarded_pr_message(&winner, None, true)
22971 .await
22972 .unwrap()
22973 .unwrap();
22974 assert_eq!(again.title, message.title);
22975 assert_eq!(again.body, message.body);
22976 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22977 assert!(
22978 !runner
22979 .state
22980 .events
22981 .iter()
22982 .any(|e| e.message.contains(&rewrite.title)
22983 || e.message.contains("English sentence."))
22984 );
22985 }
22986
22987 #[tokio::test]
22988 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22989 crate::run::pin_test_home();
22990 let dir = tempfile::tempdir().unwrap();
22991 let mut runner = runner_at(RunStatus::Gating);
22992 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22993 runner.state.repo = dir.path().to_owned();
22994 runner.state.candidates[0].worktree = dir.path().to_owned();
22995 seed_public_summary(&mut runner);
22996 let winner = runner.state.candidates[0].clone();
22997 assert!(
22998 runner
22999 .guarded_pr_message(&winner, None, true)
23000 .await
23001 .unwrap()
23002 .is_none()
23003 );
23004 let record = runner.state.github_text.as_ref().unwrap();
23005 assert!(record.rewrite.is_none());
23006 let mut old = serde_json::to_value(record).unwrap();
23008 old.as_object_mut().unwrap().remove("rewrite");
23009 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23010 assert!(old.rewrite.is_none());
23011 let id = record.question.clone().unwrap();
23012 ask::Questions::open()
23013 .update(&id, |q| {
23014 q.choices.push(crate::github_text::USE_REWRITE.into());
23015 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23016 })
23017 .unwrap();
23018 let message = runner
23019 .guarded_pr_message(&winner, None, true)
23020 .await
23021 .unwrap()
23022 .unwrap();
23023 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23024 assert_eq!(
23025 runner.state.github_text.as_ref().unwrap().choice,
23026 Some(GithubTextChoice::Fallback)
23027 );
23028 }
23029
23030 #[tokio::test]
23031 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23032 crate::run::pin_test_home();
23033 let dir = tempfile::tempdir().unwrap();
23034 let mut runner = runner_at(RunStatus::Gating);
23035 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23036 runner.state.repo = dir.path().to_owned();
23037 runner.state.candidates[0].worktree = dir.path().to_owned();
23038 seed_public_summary(&mut runner);
23039 let winner = runner.state.candidates[0].clone();
23040 assert!(
23041 runner
23042 .guarded_pr_message(&winner, None, true)
23043 .await
23044 .unwrap()
23045 .is_none()
23046 );
23047 let store = ask::Questions::open();
23048 let find = |run: &str| {
23049 store
23050 .list()
23051 .into_iter()
23052 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23053 .unwrap()
23054 };
23055 let run = runner.state.id.clone();
23056 let q = find(&run);
23057 let fp = runner
23058 .state
23059 .github_text
23060 .as_ref()
23061 .unwrap()
23062 .fingerprint
23063 .clone();
23064 let name = crate::github_text::artifact_name(&fp);
23065 let path = crate::run::artifact_path(&runner.state, &name);
23066 assert!(q.detail.contains(&path.display().to_string()));
23067 assert!(q.detail.contains("title-language: title"));
23068 let saved = std::fs::read_to_string(&path).expect("artifact written");
23069 assert!(saved.contains("日本語のタイトル"));
23070 assert!(q.cwd.is_none());
23071 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23072 let say = |id: &str, text: &str, choice: &str| {
23074 store
23075 .update(id, |q| {
23076 q.thread.push(ask::Turn {
23077 who: ask::Who::Operator,
23078 body: text.to_owned(),
23079 at: jiff::Timestamp::now(),
23080 note: None,
23081 });
23082 q.answer(ask::Answer::Choice(choice.to_owned()))
23083 })
23084 .unwrap();
23085 };
23086 say(&q.id, "まだ日本語", "use my text");
23087 assert!(
23088 runner
23089 .guarded_pr_message(&winner, None, true)
23090 .await
23091 .unwrap()
23092 .is_none()
23093 );
23094 let second = find(&run);
23095 assert_ne!(second.id, q.id);
23096 assert!(second.detail.contains("did not pass"));
23097 assert!(second.detail.contains("日本語のタイトル"));
23100 assert!(!second.detail.contains("まだ日本語"));
23101 assert!(second.detail.contains(&path.display().to_string()));
23102 say(&second.id, "fix: retry failed requests", "use my text");
23103 let message = runner
23104 .guarded_pr_message(&winner, None, true)
23105 .await
23106 .unwrap()
23107 .expect("a vetted replacement is posted");
23108 assert_eq!(message.title, "fix: retry failed requests");
23109 assert_eq!(
23110 runner
23111 .state
23112 .github_text
23113 .as_ref()
23114 .unwrap()
23115 .chosen_title
23116 .as_deref(),
23117 Some("fix: retry failed requests")
23118 );
23119 assert!(runner.state.github_text.as_ref().unwrap().resolved);
23120 let again = runner
23122 .guarded_pr_message(&winner, None, true)
23123 .await
23124 .unwrap()
23125 .unwrap();
23126 assert_eq!(again.title, message.title);
23127 assert_eq!(
23128 store
23129 .list()
23130 .into_iter()
23131 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23132 .count(),
23133 2
23134 );
23135 }
23136
23137 #[test]
23138 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23139 let mut state = state_with_summary(
23143 "add retries",
23144 "TITLE: fix(web): batch reads\n- reads run.json once",
23145 );
23146 state.config.graph.language = "ja".to_owned();
23147 let m = pr_message(&state, 'A');
23148 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23149 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23150
23151 let task = "今回やってほしいこと: results projector を直す";
23154 let mut state = state_with_summary(task, "- no title line");
23155 state.config.graph.language = "ja".to_owned();
23156 let m = pr_message(&state, 'A');
23157 assert_eq!(
23158 m.title,
23159 format!("chore: land candidate A of run {}", state.id)
23160 );
23161 assert!(
23162 m.body.contains(&format!(
23163 "<summary>Original task</summary>\n\n{task}\n\n</details>"
23164 )),
23165 "{}",
23166 m.body
23167 );
23168 }
23169
23170 #[test]
23171 fn pr_message_scrubs_home_paths_and_addresses() {
23172 let state = state_with_summary(
23173 "fix it in /Users/someone/src/x",
23174 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23175 );
23176 let m = pr_message(&state, 'A');
23177 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23178 assert!(!m.body.contains(leak), "{}", m.body);
23179 }
23180 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23181 }
23182
23183 #[test]
23184 fn pr_message_survives_a_task_that_closes_details() {
23185 let state = state_with_summary("a </details> b", "TITLE: fix: x");
23186 let m = pr_message(&state, 'A');
23187 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23188 }
23189
23190 #[test]
23191 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23192 let cmd = manual_merge_command(
23193 MergeStyle::Squash,
23194 Path::new("/repo"),
23195 "b",
23196 "fix: \"quoted\" $(x) `y`\n\nbody",
23197 );
23198 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23199 }
23200
23201 #[test]
23202 fn manual_merge_command_matches_the_configured_style() {
23203 let repo = Path::new("/repo");
23204 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23205
23206 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23207 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23208
23209 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23210 assert_eq!(
23211 squash,
23212 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23213 \"Merge magi run 0832 (candidate A)\""
23214 );
23215
23216 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23217 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23218 }
23219
23220 #[test]
23221 fn a_nudge_gets_a_quarter_of_the_budget() {
23222 assert_eq!(retry_budget(secs(1200), true), secs(300));
23224 assert_eq!(retry_budget(secs(3600), true), secs(900));
23225 }
23226
23227 #[test]
23228 fn a_resent_prompt_keeps_the_whole_budget() {
23229 assert_eq!(retry_budget(secs(1200), false), secs(1200));
23232 assert_eq!(retry_budget(secs(60), false), secs(60));
23233 }
23234
23235 #[test]
23236 fn the_floor_never_exceeds_the_original_budget() {
23237 assert_eq!(retry_budget(secs(60), true), secs(60));
23241 assert_eq!(retry_budget(secs(480), true), secs(120));
23242 assert_eq!(retry_budget(secs(0), true), secs(0));
23243 }
23244
23245 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23246 agent::CommandEvidence {
23247 id: "item1".to_owned(),
23248 description: "cargo test".to_owned(),
23249 exit_code,
23250 result_summary: String::new(),
23251 source: "codex".to_owned(),
23252 }
23253 }
23254
23255 #[test]
23256 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23257 assert!(!has_unconfirmed_command(&[]));
23261 }
23262
23263 #[test]
23264 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23265 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23269 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23270 assert!(!has_unconfirmed_command(&[
23271 evidence(Some(0)),
23272 evidence(Some(101))
23273 ]));
23274 }
23275
23276 #[test]
23277 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23278 assert!(has_unconfirmed_command(&[
23279 evidence(Some(0)),
23280 evidence(None)
23281 ]));
23282 }
23283
23284 #[test]
23285 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23286 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23287 assert_eq!(
23288 verified_noop_claim(true, &[], text).as_deref(),
23289 Some("already fixed by b32cfc4, on main.")
23290 );
23291 }
23292
23293 #[test]
23294 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23295 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23298 assert!(verified_noop_claim(false, &[], text).is_none());
23299 }
23300
23301 #[test]
23302 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23303 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23304 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23305 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23307 }
23308
23309 #[test]
23310 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23311 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23312 }
23313
23314 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23317 runner.state.candidates = shape
23318 .iter()
23319 .enumerate()
23320 .map(|(i, &(empty, verified))| Candidate {
23321 index: i,
23322 label: (b'A' + i as u8) as char,
23323 agent: "sonnet".to_owned(),
23324 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23325 worktree: PathBuf::from(format!("/wt/{i}")),
23326 summary: String::new(),
23327 stat: String::new(),
23328 files: 0,
23329 commits: 0,
23330 empty,
23331 failed: None,
23332 verified_noop: verified.map(str::to_owned),
23333 duration_ms: 0,
23334 folded: false,
23335 })
23336 .collect();
23337 }
23338
23339 #[test]
23340 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23341 ask_test_home();
23342 let mut runner = runner_at(RunStatus::Implementing);
23343 set_candidates(
23344 &mut runner,
23345 &[
23346 (true, Some("already on main at b32cfc4")),
23347 (true, Some("same fix, see the existing test")),
23348 ],
23349 );
23350
23351 runner
23352 .after_implement()
23353 .expect("a verified no-op is not an error");
23354
23355 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23356 }
23357
23358 #[test]
23359 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23360 ask_test_home();
23361 let mut runner = runner_at(RunStatus::Implementing);
23362 set_candidates(
23366 &mut runner,
23367 &[(true, Some("already on main at b32cfc4")), (true, None)],
23368 );
23369
23370 let err = runner
23371 .after_implement()
23372 .expect_err("an unverified empty candidate must still fail the run");
23373
23374 assert!(
23375 err.to_string().contains("no candidate produced a change"),
23376 "{err}"
23377 );
23378 assert_eq!(runner.state.status, RunStatus::Failed);
23379 }
23380
23381 #[test]
23382 fn after_implement_still_fails_an_ordinary_all_empty_run() {
23383 ask_test_home();
23384 let mut runner = runner_at(RunStatus::Implementing);
23385 set_candidates(&mut runner, &[(true, None), (true, None)]);
23386
23387 let err = runner
23388 .after_implement()
23389 .expect_err("no candidate declared anything; this is an ordinary failure");
23390
23391 assert!(
23392 err.to_string().contains("no candidate produced a change"),
23393 "{err}"
23394 );
23395 assert_eq!(runner.state.status, RunStatus::Failed);
23396 }
23397}
23398
23399#[derive(Debug)]
23402struct GraftRestoreFailed(String);
23403
23404impl std::fmt::Display for GraftRestoreFailed {
23405 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23406 write!(
23407 f,
23408 "could not restore the winner to its pre-graft commit: {}",
23409 self.0
23410 )
23411 }
23412}
23413
23414impl std::error::Error for GraftRestoreFailed {}
23415
23416#[cfg(test)]
23420mod resource_wait_tests {
23421 use super::*;
23422 use std::collections::BTreeMap;
23423
23424 fn sh(id: &str, script: &str) -> AgentSpec {
23425 AgentSpec {
23426 id: id.to_owned(),
23427 kind: crate::config::AgentKind::Command,
23428 model: None,
23429 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23430 extra_args: Vec::new(),
23431 env: BTreeMap::new(),
23432 prompt_delivery: None,
23433 max_turn_secs: None,
23434 }
23435 }
23436
23437 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23438 SeatJob {
23439 seat: SeatState::new("impl-A", &spec.id, 1),
23440 spec,
23441 cwd: dir.to_path_buf(),
23442 prompt: "do it".to_owned(),
23443 timeout,
23444 allow_write: true,
23445 sessions: false,
23446 artifacts: dir.join("art"),
23447 stem: "impl-A".to_owned(),
23448 handover: None,
23449 }
23450 }
23451
23452 fn state(resource_wait_secs: u64) -> RunState {
23453 let mut config = Config::default();
23454 config.graph.timeout_resource_wait = resource_wait_secs;
23455 RunState::new(
23456 PathBuf::from("/repo"),
23457 "main".to_owned(),
23458 "abc1234".to_owned(),
23459 "task".to_owned(),
23460 config,
23461 )
23462 }
23463
23464 fn hold(cache: &Path) -> crate::cache::Guard {
23465 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23466 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23467 crate::cache::AcquireOutcome::Acquired(g) => g,
23468 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23469 }
23470 }
23471
23472 async fn run(
23473 jobs: Vec<SeatJob>,
23474 sem: Arc<Semaphore>,
23475 cache: Option<&Path>,
23476 st: &mut RunState,
23477 ) -> Vec<(usize, SeatState, AgentOutcome)> {
23478 let prompts = Prompts::default();
23479 let ctx = WaveCtx {
23480 run: "r",
23481 node: "implement",
23482 prompts: &prompts,
23483 cache,
23484 round: None,
23485 carry_seats: false,
23486 pauses: None,
23487 };
23488 wave(jobs, sem, &ctx, st, 0).await
23489 }
23490
23491 #[tokio::test]
23492 async fn verification_cache_wait_preserves_the_command_budget() {
23493 crate::run::pin_test_home();
23494 let tmp = tempfile::tempdir().unwrap();
23495 let cache = tmp.path().join("target");
23496 let guard = hold(&cache);
23497 let release = tokio::spawn(async move {
23498 tokio::time::sleep(Duration::from_millis(100)).await;
23499 guard.release();
23500 });
23501 let mut st = state(10);
23502 let budget = Duration::from_millis(50);
23503 let wait = repository_wait(&st, None);
23504 let (outcomes, _) = with_cache_lease(
23505 &mut st,
23506 Some(&cache),
23507 "gate",
23508 "gate",
23509 tmp.path(),
23510 "abc1234",
23511 budget,
23512 "test gate",
23513 &wait,
23514 |_, available| async move {
23515 assert_eq!(
23516 available, budget,
23517 "contention must not shorten verification"
23518 );
23519 (Vec::new(), false, Vec::new())
23520 },
23521 )
23522 .await
23523 .unwrap();
23524 release.await.unwrap();
23525 assert!(outcomes.is_empty(), "{outcomes:?}");
23526 assert!(
23527 st.events
23528 .iter()
23529 .any(|event| event.message.contains("waiting for"))
23530 );
23531 assert!(st.resource_wait.is_none());
23532 hold(&cache).release();
23534 }
23535
23536 #[tokio::test]
23537 async fn verification_cache_wait_expires_without_starting_commands() {
23538 crate::run::pin_test_home();
23539 let tmp = tempfile::tempdir().unwrap();
23540 let cache = tmp.path().join("target");
23541 let _held = hold(&cache);
23542 let mut st = state(0);
23543 let wait = repository_wait(&st, None);
23544 let (outcomes, _) = with_cache_lease(
23545 &mut st,
23546 Some(&cache),
23547 "gate",
23548 "gate",
23549 tmp.path(),
23550 "abc1234",
23551 Duration::from_secs(60),
23552 "test gate",
23553 &wait,
23554 |_, _| async { panic!("commands must not start without the cache lease") },
23555 )
23556 .await
23557 .unwrap();
23558 assert_eq!(outcomes.len(), 1);
23559 assert!(outcomes[0].resource_blocked);
23560 assert!(st.resource_wait.is_none());
23561 assert!(matches!(
23562 st.resource_block.unwrap().cause,
23563 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23564 ));
23565 }
23566
23567 #[tokio::test]
23568 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23569 crate::run::pin_test_home();
23570 let tmp = tempfile::tempdir().unwrap();
23571 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23572 let cache = tmp.path().join("cache");
23573 let mut state = state(21600);
23574 let wait = repository_wait(&state, None);
23575 let (outcomes, _) = with_cache_lease(
23576 &mut state,
23577 Some(&cache),
23578 "gate",
23579 "gate",
23580 tmp.path(),
23581 "head",
23582 Duration::from_secs(60),
23583 "gate",
23584 &wait,
23585 |_, _| async { panic!("freshness failure must not start verification") },
23586 )
23587 .await
23588 .unwrap();
23589 assert_eq!(outcomes.len(), 1);
23590 assert!(outcomes[0].resource_blocked);
23591 let block = state.resource_block.unwrap();
23592 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23593 assert!(!block.description().contains("timed out"));
23594 }
23595
23596 #[tokio::test]
23597 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23598 crate::run::pin_test_home();
23599 let tmp = tempfile::tempdir().unwrap();
23600 let cache = tmp.path().join("cache");
23601 let _held = hold(&cache);
23602 let mut state = state(21600);
23603 let pause = Pause::new();
23604 let interrupt = Pause::new();
23605 let cause = ParkCause::Operator;
23606 pause.park_for(cause.clone());
23607 let prompts = Prompts::default();
23608 let ctx = WaveCtx {
23609 run: "test",
23610 node: "fix",
23611 prompts: &prompts,
23612 cache: Some(&cache),
23613 round: None,
23614 carry_seats: false,
23615 pauses: Some((&pause, &interrupt)),
23616 };
23617 let result = wave(
23618 vec![job(
23619 sh("writer", "exit 99"),
23620 tmp.path(),
23621 Duration::from_secs(1),
23622 )],
23623 Arc::new(Semaphore::new(1)),
23624 &ctx,
23625 &mut state,
23626 0,
23627 )
23628 .await;
23629 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23630 assert_eq!(result[0].1.turns, 0);
23631 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23632 assert_eq!(
23633 state.resource_block.unwrap().cause,
23634 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23635 );
23636 assert!(state.active.is_empty());
23637 }
23638
23639 #[tokio::test]
23640 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23641 crate::run::pin_test_home();
23642 let tmp = tempfile::tempdir().unwrap();
23643 let cache = tmp.path().join("target-a");
23644 let guard = hold(&cache);
23645 tokio::spawn(async move {
23647 tokio::time::sleep(Duration::from_millis(1500)).await;
23648 guard.release();
23649 });
23650 let mut st = state(60);
23651 let results = run(
23652 vec![job(
23653 sh("alpha", "echo ok"),
23654 tmp.path(),
23655 Duration::from_secs(1),
23656 )],
23657 Arc::new(Semaphore::new(1)),
23658 Some(&cache),
23659 &mut st,
23660 )
23661 .await;
23662 assert!(
23663 matches!(results[0].2, AgentOutcome::Ok(_)),
23664 "the wait must not be charged to the agent"
23665 );
23666 assert!(st.resource_block.is_none());
23667 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23668 assert!(st.resource_wait.is_none() && st.active.is_empty());
23669 }
23670
23671 #[tokio::test]
23672 #[cfg(unix)]
23673 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23674 use std::os::fd::AsRawFd as _;
23675 crate::run::pin_test_home();
23676 let tmp = tempfile::tempdir().unwrap();
23677 crate::git::git(tmp.path(), &["init"]).await.unwrap();
23678 let common = tmp.path().join(".git");
23679 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23680 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23681 assert_eq!(
23682 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23683 0
23684 );
23685 let mut st = state(1);
23686 st.config.coordination = crate::config::test_coordination();
23687 let mut reader = job(
23688 sh("reader", "echo must-not-run"),
23689 tmp.path(),
23690 Duration::from_secs(30),
23691 );
23692 reader.allow_write = false;
23693 let results = run(
23694 vec![reader.clone()],
23695 Arc::new(Semaphore::new(1)),
23696 None,
23697 &mut st,
23698 )
23699 .await;
23700 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23701 assert_eq!(results[0].1.turns, 0);
23702 assert!(FailClass::of(&results[0].2).is_none());
23703 assert_eq!(
23704 st.resource_block.as_ref().unwrap().kind,
23705 "repository coordination"
23706 );
23707 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23708 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23709 drop(external);
23710 let editor = crate::repo_lock::SeatLease::for_checkout(
23711 tmp.path(),
23712 true,
23713 crate::config::test_coordination().as_ref(),
23714 )
23715 .await
23716 .unwrap();
23717 let pause = Pause::new();
23718 let interrupt = Pause::new();
23719 let prompts = Prompts::default();
23720 let ctx = WaveCtx {
23721 run: "r",
23722 node: "review",
23723 prompts: &prompts,
23724 cache: None,
23725 round: None,
23726 carry_seats: false,
23727 pauses: Some((&pause, &interrupt)),
23728 };
23729 let (results, ()) = tokio::join!(
23730 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23731 async {
23732 tokio::time::sleep(Duration::from_millis(50)).await;
23733 pause.park();
23734 }
23735 );
23736 assert!(
23737 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23738 );
23739 assert_eq!(results[0].1.turns, 0);
23740 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23741 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23742 drop(editor);
23743 }
23744
23745 #[tokio::test]
23746 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23747 crate::run::pin_test_home();
23748 let tmp = tempfile::tempdir().unwrap();
23749 let cache = tmp.path().join("target-b");
23750 let _held = hold(&cache);
23751 let mut st = state(1);
23752 let results = run(
23753 vec![job(
23754 sh("alpha", "echo ok"),
23755 tmp.path(),
23756 Duration::from_secs(1),
23757 )],
23758 Arc::new(Semaphore::new(1)),
23759 Some(&cache),
23760 &mut st,
23761 )
23762 .await;
23763 let (_, seat, out) = &results[0];
23764 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23765 assert!(wave_blocked(&results));
23766 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23767 assert_eq!(seat.turns, 0, "nothing was asked");
23768 assert!(
23769 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23770 "no agent was spawned"
23771 );
23772 assert!(FailClass::of(out).is_none(), "not a failure class");
23773 let block = st.resource_block.as_ref().expect("recorded for the resume");
23774 assert_eq!(block.kind, "shared build cache");
23775 assert!(matches!(
23776 block.cause,
23777 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23778 ));
23779 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23780 assert!(st.active.is_empty());
23781 }
23782
23783 #[tokio::test]
23784 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23785 crate::run::pin_test_home();
23786 let tmp = tempfile::tempdir().unwrap();
23787 let mut st = state(1);
23788 let results = run(
23789 vec![job(
23790 sh("alpha", "echo ok"),
23791 tmp.path(),
23792 Duration::from_secs(1),
23793 )],
23794 Arc::new(Semaphore::new(0)),
23795 None,
23796 &mut st,
23797 )
23798 .await;
23799 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23800 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23801 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23802 }
23803
23804 #[tokio::test]
23805 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23806 crate::run::pin_test_home();
23807 let tmp = tempfile::tempdir().unwrap();
23808 let mut st = state(60);
23809 let results = run(
23810 vec![job(
23811 sh("alpha", "sleep 30"),
23812 tmp.path(),
23813 Duration::from_secs(1),
23814 )],
23815 Arc::new(Semaphore::new(1)),
23816 None,
23817 &mut st,
23818 )
23819 .await;
23820 let out = &results[0].2;
23821 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23822 let class = FailClass::of(out).expect("a failure");
23823 assert_eq!(class, FailClass::Timeout);
23824 assert!(should_hand_over(None, &class), "existing handover policy");
23825 assert!(st.resource_block.is_none());
23826 }
23827}